Use statefulset for runner.

deploy/crds/vms-crd.yaml

@@ -178,39 +178,12 @@ spec:
                               (for whatever reason) is to use a label selector alongside 
                               manually created PersistentVolumes.
                             properties:
-                              apiVersion:
-                                description: >-
-                                  APIVersion defines the versioned schema of this
-                                  representation of an object. Servers should convert recognized
-                                  schemas to the latest internal value, and may reject unrecognized
-                                  values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
-                                type: string
-                                default: v1
-                              kind:
-                                description: >-
-                                  Kind is a string value representing the REST
-                                  resource this object represents. Servers may infer this
-                                  from the endpoint the client submits requests to. Cannot
-                                  be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
-                                type: string
-                                default: PersistentVolumeClaim
                               metadata:
                                 description: >-
                                   EmbeddedMetadata contains metadata relevant to
                                   an EmbeddedResource.
                                 type: object
                                 properties:
-                                  namespace:
-                                    description: >-
-                                      Namespace defines the space within which each 
-                                      name must be unique. An empty namespace is equivalent to the 
-                                      "default" namespace, but "default" is the canonical 
-                                      representation. Not all objects are required to be scoped 
-                                      to a namespace - the value of this field for those objects 
-                                      will be empty. Must be a DNS_LABEL. Cannot be updated. 
-                                      More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces.
-                                      The default value is the VM's namespace.
-                                    type: string
                                   name:
                                     description: >-
                                       Name must be unique within a namespace.

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml

@@ -139,13 +139,13 @@ data:
         <#if disk.volumeClaimTemplate?? 
           && disk.volumeClaimTemplate.metadata??
           && disk.volumeClaimTemplate.metadata.name??>
-          <#assign name = disk.volumeClaimTemplate.metadata.name.asString>
+          <#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
         <#else>
-          <#assign name = "" + drvCounter>
+          <#assign diskName = "disk-" + drvCounter>
         </#if>
         <#if disk.volumeClaimTemplate??> 
         - type: raw
-          resource: /dev/disk-${ name }
+          resource: /dev/${ diskName }
           <#if disk.bootindex??>
           bootindex: ${ disk.bootindex.asInt?c }
           </#if>

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml

@@ -1,16 +0,0 @@
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  namespace: ${ cr.metadata.namespace.asString }
-  name: ${ cr.metadata.name.asString + "-runner-data" }
-  labels:
-    app.kubernetes.io/name: ${ constants.APP_NAME }
-    app.kubernetes.io/instance: ${ cr.metadata.name.asString }
-    app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
-    
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Mi

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml

@@ -1,94 +0,0 @@
-kind: Pod
-apiVersion: v1
-metadata:
-  namespace: ${ cr.metadata.namespace.asString }
-  name: ${ cr.metadata.name.asString }
-  labels:
-    app.kubernetes.io/name: ${ constants.APP_NAME }
-    app.kubernetes.io/instance: ${ cr.metadata.name.asString }
-    app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
-  annotations:
-    # Triggers update of config map mounted in pod
-    # See https://ahmet.im/blog/kubernetes-secret-volumes-delay/
-    vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }"
-
-spec:
-  containers:
-  - name: ${ cr.metadata.name.asString }
-    <#assign image = cr.spec.image>
-    image: ${ image.repository.asString }/${ image.path.asString }:${ image.version.asString }
-    resources: {}
-    imagePullPolicy: ${ image.pullPolicy.asString }
-    volumeMounts:
-    # Not needed because pod is priviledged:
-    # - mountPath: /dev/kvm
-    #   name: dev-kvm
-    # - mountPath: /dev/net/tun
-    #   name: dev-tun
-    # - mountPath: /sys/fs/cgroup
-    #   name: cgroup
-    - name: config
-      mountPath: /etc/opt/vmrunner
-    - name: vm-data
-      mountPath: /var/local/vm-data
-    - name: vmop-image-repository
-      mountPath: ${ constants.IMAGE_REPO_PATH }
-    volumeDevices:
-    <#assign diskCounter = 0/>
-    <#list cr.spec.vm.disks.asList() as disk>
-    <#if disk.volumeClaimTemplate??>
-    <#if disk.volumeClaimTemplate.metadata??
-      && disk.volumeClaimTemplate.metadata.name??>
-      <#assign diskName = "disk-" + disk.volumeClaimTemplate.metadata.name.asString>
-    <#else>
-      <#assign diskName = "disk-" + diskCounter>
-    </#if>
-    - name: ${ diskName }
-      devicePath: /dev/${ diskName }
-    <#assign diskCounter = diskCounter + 1/>
-    </#if>
-    </#list>
-    securityContext:
-      privileged: true
-  volumes:
-  # Not needed because pod is priviledged:
-  # - name: dev-kvm
-  #   hostPath:
-  #     path: /dev/kvm
-  #     type: CharDevice
-  # - hostPath:
-  #     path: /dev/net/tun
-  #     type: CharDevice
-  #   name: dev-tun
-  # - name: cgroup
-  #   hostPath:
-  #     path: /sys/fs/cgroup
-  - name: config
-    configMap:
-      name: ${ cr.metadata.name.asString }
-  - name: vm-data
-    persistentVolumeClaim:
-      claimName: ${ cr.metadata.name.asString }-runner-data
-  - name: vmop-image-repository
-    persistentVolumeClaim:
-      claimName: vmop-image-repository
-  <#assign diskCounter = 0/>
-  <#list cr.spec.vm.disks.asList() as disk>
-  <#if disk.volumeClaimTemplate??>
-  <#if disk.volumeClaimTemplate.metadata??
-    && disk.volumeClaimTemplate.metadata.name??>
-    <#assign claimName = disk.volumeClaimTemplate.metadata.name.asString>
-    <#assign diskName = "disk-" + claimName>
-  <#else>
-    <#assign claimName = cr.metadata.name.asString + "-disk-" + diskCounter>
-    <#assign diskName = "disk-" + diskCounter>
-  </#if>
-  - name: ${ diskName }
-    persistentVolumeClaim:
-      claimName: ${ claimName }
-  <#assign diskCounter = diskCounter + 1/>
-  </#if>
-  </#list>
-  hostNetwork: true
-  terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
-  restartPolicy: Never

org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerSts.ftl.yaml

@@ -0,0 +1,126 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  namespace: ${ cr.metadata.namespace.asString }
+  name: ${ cr.metadata.name.asString }
+  labels:
+    app.kubernetes.io/name: ${ constants.APP_NAME }
+    app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+    app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ${ constants.APP_NAME }
+      app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+  replicas: ${ (cr.spec.vm.state.asString == "Running")?then(1, 0) }
+  template:
+    metadata:
+      namespace: ${ cr.metadata.namespace.asString }
+      name: ${ cr.metadata.name.asString }
+      labels:
+        app.kubernetes.io/name: ${ constants.APP_NAME }
+        app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+        app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
+      annotations:
+        # Triggers update of config map mounted in pod
+        # See https://ahmet.im/blog/kubernetes-secret-volumes-delay/
+        vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }"
+    spec:
+      containers:
+      - name: ${ cr.metadata.name.asString }
+        <#assign image = cr.spec.image>
+        image: ${ image.repository.asString }/${ image.path.asString }:${ image.version.asString }
+        resources: {}
+        imagePullPolicy: ${ image.pullPolicy.asString }
+        volumeMounts:
+        # Not needed because pod is priviledged:
+        # - mountPath: /dev/kvm
+        #   name: dev-kvm
+        # - mountPath: /dev/net/tun
+        #   name: dev-tun
+        # - mountPath: /sys/fs/cgroup
+        #   name: cgroup
+        - name: config
+          mountPath: /etc/opt/vmrunner
+        - name: runner-data
+          mountPath: /var/local/vm-data
+        - name: vmop-image-repository
+          mountPath: ${ constants.IMAGE_REPO_PATH }
+        volumeDevices:
+        <#assign diskCounter = 0/>
+        <#list cr.spec.vm.disks.asList() as disk>
+        <#if disk.volumeClaimTemplate??>
+        <#if disk.volumeClaimTemplate.metadata??
+          && disk.volumeClaimTemplate.metadata.name??>
+          <#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
+        <#else>
+          <#assign diskName = "disk-" + diskCounter>
+        </#if>
+        - name: ${ diskName }
+          devicePath: /dev/${ diskName }
+        <#assign diskCounter = diskCounter + 1/>
+        </#if>
+        </#list>
+        securityContext:
+          privileged: true
+      volumes:
+      # Not needed because pod is priviledged:
+      # - name: dev-kvm
+      #   hostPath:
+      #     path: /dev/kvm
+      #     type: CharDevice
+      # - hostPath:
+      #     path: /dev/net/tun
+      #     type: CharDevice
+      #   name: dev-tun
+      # - name: cgroup
+      #   hostPath:
+      #     path: /sys/fs/cgroup
+      - name: config
+        configMap:
+          name: ${ cr.metadata.name.asString }
+      - name: vmop-image-repository
+        persistentVolumeClaim:
+          claimName: vmop-image-repository
+      hostNetwork: true
+      terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
+  volumeClaimTemplates:
+  - metadata:
+      namespace: ${ cr.metadata.namespace.asString }
+      name: runner-data
+      labels:
+        app.kubernetes.io/name: ${ constants.APP_NAME }
+        app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+        app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Mi
+  <#assign diskCounter = 0/>
+  <#list cr.spec.vm.disks.asList() as disk>
+  <#if disk.volumeClaimTemplate??>
+  <#if disk.volumeClaimTemplate.metadata??
+    && disk.volumeClaimTemplate.metadata.name??>
+    <#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
+  <#else>
+    <#assign diskName = "disk-" + diskCounter>
+  </#if>
+  - metadata:
+      namespace: ${ cr.metadata.namespace.asString }
+      name: ${ diskName }
+      labels:
+        app.kubernetes.io/name: ${ constants.APP_NAME }
+        app.kubernetes.io/instance: ${ cr.metadata.name.asString }
+        app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
+      <#if disk.volumeClaimTemplate.metadata??
+        && disk.volumeClaimTemplate.metadata.annotations??>
+      annotations:
+        ${ disk.volumeClaimTemplate.metadata.annotations.toString() }
+      </#if>
+    spec:
+      ${ disk.volumeClaimTemplate.spec.toString() }
+  <#assign diskCounter = diskCounter + 1/>
+  </#if>
+  </#list>

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DataReconciler.java

@@ -1,86 +0,0 @@
-/*
- * VM-Operator
- * Copyright (C) 2023 Michael N. Lipp
- * 
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.jdrupes.vmoperator.manager;
-
-import com.google.gson.JsonObject;
-import freemarker.template.Configuration;
-import freemarker.template.TemplateException;
-import io.kubernetes.client.openapi.ApiException;
-import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
-import io.kubernetes.client.util.generic.dynamic.Dynamics;
-import java.io.IOException;
-import java.io.StringWriter;
-import java.util.Map;
-
-/**
- * Delegee for reconciling the data PVC
- */
-@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
-/* default */ class DataReconciler {
-
-    private final Configuration fmConfig;
-
-    /**
-     * Instantiates a new config map reconciler.
-     *
-     * @param fmConfig the fm config
-     */
-    public DataReconciler(Configuration fmConfig) {
-        this.fmConfig = fmConfig;
-    }
-
-    /**
-     * Reconcile.
-     *
-     * @param model the model
-     * @param channel the channel
-     * @throws TemplateException the template exception
-     * @throws ApiException the api exception
-     * @throws IOException Signals that an I/O exception has occurred.
-     */
-    @SuppressWarnings("PMD.ConfusingTernary")
-    public void reconcile(Map<String, Object> model, VmChannel channel)
-            throws TemplateException, ApiException, IOException {
-        // Combine template and data and parse result
-        var fmTemplate = fmConfig.getTemplate("runnerDataPvc.ftl.yaml");
-        StringWriter out = new StringWriter();
-        fmTemplate.process(model, out);
-        // Avoid Yaml.load due to
-        // https://github.com/kubernetes-client/java/issues/2741
-        var pvcDef = Dynamics.newFromYaml(out.toString());
-
-        // Get API and check if PVC exists
-        DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
-            "persistentvolumeclaims", channel.client());
-        var existing = K8s.get(pvcApi, pvcDef.getMetadata());
-
-        // If PVC does not exist, create. Else patch (apply)
-        if (existing.isEmpty()) {
-            pvcApi.create(pvcDef);
-        } else {
-            // spec is immutable, so mix in existing spec
-            GsonPtr.to(pvcDef.getRaw()).set("spec", GsonPtr
-                .to(existing.get().getRaw()).get(JsonObject.class, "spec")
-                .get().deepCopy());
-            K8s.apply(pvcApi, existing.get(),
-                channel.client().getJSON().serialize(pvcDef));
-        }
-    }
-
-}

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisksReconciler.java

@@ -1,128 +0,0 @@
-/*
- * VM-Operator
- * Copyright (C) 2023 Michael N. Lipp
- * 
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.jdrupes.vmoperator.manager;
-
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonPrimitive;
-import io.kubernetes.client.openapi.ApiException;
-import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
-import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
-import io.kubernetes.client.util.generic.options.ListOptions;
-import java.util.Collections;
-import static org.jdrupes.vmoperator.manager.Constants.APP_NAME;
-import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME;
-
-/**
- * Delegee for reconciling the PVCs for the disks
- */
-@SuppressWarnings("PMD.DataflowAnomalyAnalysis")
-/* default */ class DisksReconciler {
-
-    /**
-     * Reconcile disks.
-     *
-     * @param vmDef the vm def
-     * @param channel the channel
-     * @throws ApiException the api exception
-     */
-    public void reconcile(JsonObject vmDef,
-            VmChannel channel) throws ApiException {
-        @SuppressWarnings("PMD.AvoidDuplicateLiterals")
-        var disks = GsonPtr.to(vmDef)
-            .get(JsonArray.class, "spec", "vm", "disks")
-            .map(JsonArray::asList).orElse(Collections.emptyList());
-        int index = 0;
-        for (var disk : disks) {
-            reconcileDisk(vmDef, index++, (JsonObject) disk, channel);
-        }
-    }
-
-    @SuppressWarnings({ "PMD.AvoidDuplicateLiterals", "PMD.ConfusingTernary" })
-    private void reconcileDisk(JsonObject vmDefinition,
-            int index, JsonObject diskDef, VmChannel channel)
-            throws ApiException {
-        if (!diskDef.has("volumeClaimTemplate")) {
-            return;
-        }
-        var pvcObject = new DynamicKubernetesObject();
-        var pvcRaw = GsonPtr.to(pvcObject.getRaw());
-        var vmRaw = GsonPtr.to(vmDefinition);
-        var pvcTpl = GsonPtr.to(diskDef).to("volumeClaimTemplate");
-
-        // Copy base and metadata from template and add missing/additional data.
-        pvcObject.setApiVersion(pvcTpl.getAsString("apiVersion").get());
-        pvcObject.setKind(pvcTpl.getAsString("kind").get());
-        var vmName = vmRaw.getAsString("metadata", "name").orElse("default");
-        pvcRaw.get(JsonObject.class).add("metadata",
-            pvcTpl.to("metadata").get(JsonObject.class).deepCopy());
-        var defMeta = pvcRaw.to("metadata");
-        defMeta.computeIfAbsent("namespace", () -> new JsonPrimitive(
-            vmRaw.getAsString("metadata", "namespace").orElse("default")));
-        defMeta.computeIfAbsent("name", () -> new JsonPrimitive(
-            vmName + "-disk-" + index));
-        var pvcLbls = pvcRaw.to("metadata", "labels");
-        pvcLbls.set("app.kubernetes.io/name", APP_NAME);
-        pvcLbls.set("app.kubernetes.io/instance", vmName);
-        pvcLbls.set("app.kubernetes.io/component", "disk");
-        pvcLbls.set("app.kubernetes.io/managed-by", VM_OP_NAME);
-
-        // Get API and check if PVC exists
-        DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
-            "persistentvolumeclaims", channel.client());
-        var existing = K8s.get(pvcApi, pvcObject.getMetadata());
-
-        // If PVC does not exist, create. Else patch (apply)
-        if (existing.isEmpty()) {
-            // PVC does not exist yet, copy spec from template
-            pvcRaw.get(JsonObject.class).add("spec",
-                pvcTpl.to("spec").get(JsonObject.class).deepCopy());
-            pvcApi.create(pvcObject);
-        } else {
-            // spec is immutable, so mix in existing spec
-            pvcRaw.set("spec", GsonPtr.to(existing.get().getRaw())
-                .to("spec").get().deepCopy());
-            K8s.apply(pvcApi, existing.get(),
-                channel.client().getJSON().serialize(pvcObject));
-        }
-    }
-
-    /**
-     * Delete the PVCs generated from the defined disks.
-     *
-     * @param event the event
-     * @param channel the channel
-     * @throws ApiException the api exception
-     */
-    public void deleteDisks(VmDefChanged event, VmChannel channel)
-            throws ApiException {
-        // Get API and check and list related
-        var pvcApi = K8s.pvcApi(channel.client());
-        var pvcs = pvcApi.list(event.object().getMetadata().getNamespace(),
-            new ListOptions().labelSelector(
-                "app.kubernetes.io/managed-by=" + VM_OP_NAME
-                    + ",app.kubernetes.io/name=" + APP_NAME
-                    + ",app.kubernetes.io/instance="
-                    + event.object().getMetadata().getName()));
-        for (var pvc : pvcs.getObject().getItems()) {
-            K8s.delete(pvcApi, pvc);
-        }
-    }
-
-}

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java

@@ -50,9 +50,7 @@ public class Reconciler extends Component {
     @SuppressWarnings("PMD.SingularField")
     private final Configuration fmConfig;
     private final CmReconciler cmReconciler;
-    private final DataReconciler dataReconciler;
+    private final StsReconciler stsReconciler;
-    private final DisksReconciler disksReconciler;
-    private final PodReconciler podReconciler;
 
     /**
      * Instantiates a new reconciler.
@@ -73,9 +71,7 @@ public class Reconciler extends Component {
         fmConfig.setClassForTemplateLoading(Reconciler.class, "");
 
         cmReconciler = new CmReconciler(fmConfig);
-        disksReconciler = new DisksReconciler();
+        stsReconciler = new StsReconciler(fmConfig);
-        dataReconciler = new DataReconciler(fmConfig);
-        podReconciler = new PodReconciler(fmConfig);
     }
 
     /**
@@ -120,15 +116,12 @@ public class Reconciler extends Component {
 
         // Reconcile
         if (event.type() != Type.DELETED) {
-            dataReconciler.reconcile(model, channel);
-            disksReconciler.reconcile(vmDef, channel);
             var configMap = cmReconciler.reconcile(event, model, channel);
             model.put("cm", configMap.getRaw());
-            podReconciler.reconcile(event, model, channel);
+            stsReconciler.reconcile(event, model, channel);
         } else {
-            podReconciler.reconcile(event, model, channel);
+            stsReconciler.reconcile(event, model, channel);
             cmReconciler.reconcile(event, model, channel);
-            disksReconciler.deleteDisks(event, channel);
         }
     }
 

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StsReconciler.java

@@ -25,18 +25,17 @@ import io.kubernetes.client.custom.V1Patch;
 import io.kubernetes.client.openapi.ApiException;
 import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
 import io.kubernetes.client.util.generic.dynamic.Dynamics;
-import io.kubernetes.client.util.generic.options.DeleteOptions;
+import io.kubernetes.client.util.generic.options.PatchOptions;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.Map;
-import static org.jdrupes.vmoperator.manager.Constants.STATE_STOPPED;
 import org.jdrupes.vmoperator.manager.VmDefChanged.Type;
 
 /**
  * Delegee for reconciling the pod.
  */
 @SuppressWarnings("PMD.DataflowAnomalyAnalysis")
-/* default */ class PodReconciler {
+/* default */ class StsReconciler {
 
     private final Configuration fmConfig;
 
@@ -45,12 +44,12 @@ import org.jdrupes.vmoperator.manager.VmDefChanged.Type;
      *
      * @param fmConfig the fm config
      */
-    public PodReconciler(Configuration fmConfig) {
+    public StsReconciler(Configuration fmConfig) {
         this.fmConfig = fmConfig;
     }
 
     /**
-     * Reconcile pod.
+     * Reconcile stateful set.
      *
      * @param event the event
      * @param model the model
@@ -63,51 +62,38 @@ import org.jdrupes.vmoperator.manager.VmDefChanged.Type;
             VmChannel channel)
             throws IOException, TemplateException, ApiException {
         // Check if exists
-        DynamicKubernetesApi podApi = new DynamicKubernetesApi("", "v1",
+        DynamicKubernetesApi stsApi = new DynamicKubernetesApi("apps", "v1",
-            "pods", channel.client());
+            "statefulsets", channel.client());
-        var existing = K8s.get(podApi, event.object().getMetadata());
+        // var existing = K8s.get(stsApi, event.object().getMetadata());
 
-        // Get desired state.
+        if (event.type() == Type.DELETED) {
-        var delete = event.type() == Type.DELETED
+            var meta = GsonPtr.to((JsonObject) model.get("cr")).to("metadata");
-            || GsonPtr.to((JsonObject) model.get("cr")).to("spec", "vm")
+            PatchOptions opts = new PatchOptions();
-                .getAsString("state").orElse("").equals(STATE_STOPPED);
+            opts.setFieldManager("kubernetes-java-kubectl-apply");
-
+            stsApi.patch(meta.getAsString("namespace").get(),
-        // If deleted or stopped, delete
+                meta.getAsString("name").get(), V1Patch.PATCH_FORMAT_JSON_PATCH,
-        if (delete) {
+                new V1Patch("[{\"op\": \"replace\", "
-            if (existing.isPresent()) {
+                    + "\"path\": \"/spec/replicas\", \"value\": 0}]"),
-                var opts = new DeleteOptions();
+                opts).throwsApiException();
-                opts.setGracePeriodSeconds(
+            stsApi.delete(meta.getAsString("namespace").get(),
-                    GsonPtr.to((JsonObject) model.get("cr")).to("spec", "vm")
+                meta.getAsString("name").get()).throwsApiException();
-                        .getAsLong("powerdownTimeout").get() + 1);
-                K8s.delete(podApi, existing.get(), opts);
-            }
             return;
         }
 
         // Combine template and data and parse result
-        var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml");
+        var fmTemplate = fmConfig.getTemplate("runnerSts.ftl.yaml");
         StringWriter out = new StringWriter();
         fmTemplate.process(model, out);
         // Avoid Yaml.load due to
         // https://github.com/kubernetes-client/java/issues/2741
-        var podDef = Dynamics.newFromYaml(out.toString());
+        var stsDef = Dynamics.newFromYaml(out.toString());
-
+        PatchOptions opts = new PatchOptions();
-        // Check if update
+        opts.setForce(false);
-        if (existing.isEmpty()) {
+        opts.setFieldManager("kubernetes-java-kubectl-apply");
-            podApi.create(podDef);
+        stsApi.patch(stsDef.getMetadata().getNamespace(),
-        } else {
+            stsDef.getMetadata().getName(), V1Patch.PATCH_FORMAT_APPLY_YAML,
-            // only annotations are updated
+            new V1Patch(channel.client().getJSON().serialize(stsDef)),
-            var metadata = new JsonObject();
+            opts).throwsApiException();
-            metadata.add("annotations", GsonPtr.to(podDef.getRaw())
-                .to("metadata").get(JsonObject.class, "annotations").get());
-            var patch = new JsonObject();
-            patch.add("metadata", metadata);
-            podApi.patch(existing.get().getMetadata().getNamespace(),
-                existing.get().getMetadata().getName(),
-                V1Patch.PATCH_FORMAT_JSON_MERGE_PATCH,
-                new V1Patch(channel.client().getJSON().serialize(patch)))
-                .throwsApiException();
-        }
     }
 
 }

org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmWatcher.java

@@ -34,6 +34,7 @@ import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
 import io.kubernetes.client.util.generic.options.ListOptions;
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -100,6 +101,8 @@ public class VmWatcher extends Component {
         }
     }
 
+    @SuppressWarnings({ "PMD.AvoidInstantiatingObjectsInLoops",
+        "PMD.CognitiveComplexity" })
     private void purge(CustomObjectsApi coa, List<String> vmOpApiVersions)
             throws ApiException {
         // Get existing CRs (VMs)
@@ -117,20 +120,28 @@ public class VmWatcher extends Component {
         opts.setLabelSelector(
             "app.kubernetes.io/managed-by=vmoperator,"
                 + "app.kubernetes.io/name=vmrunner");
-        for (var version : vmOpApiVersions) {
+        for (String resource : List.of("apps/v1/statefulsets",
-            for (String resource : List.of("pods", "configmaps",
+            "v1/persistentvolumeclaims", "v1/configmaps", "v1/secrets")) {
-                "persistentvolumeclaims", "secrets")) {
+            var resParts = new LinkedList<>(List.of(resource.split("/")));
-                // Get resources, selected by label
+            var group = resParts.size() == 3 ? resParts.poll() : "";
-                @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
+            var version = resParts.poll();
-                var api
+            var plural = resParts.poll();
-                    = new DynamicKubernetesApi("", version, resource, client);
+            // Get resources, selected by label
-                for (var obj : api.list(managedNamespace, opts).getObject()
+            @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
-                    .getItems()) {
+            var api = new DynamicKubernetesApi(group, version, plural, client);
-                    String instance = obj.getMetadata().getLabels()
+            var listObj = api.list(managedNamespace, opts).getObject();
-                        .get("app.kubernetes.io/instance");
+            if (listObj == null) {
-                    if (!known.contains(instance)) {
+                continue;
-                        api.delete(managedNamespace,
+            }
-                            obj.getMetadata().getName());
+            for (var obj : listObj.getItems()) {
+                String instance = obj.getMetadata().getLabels()
+                    .get("app.kubernetes.io/instance");
+                if (!known.contains(instance)) {
+                    var resName = obj.getMetadata().getName();
+                    var result = api.delete(managedNamespace, resName);
+                    if (!result.isSuccess()) {
+                        logger.warning(() -> "Cannot cleanup resource \""
+                            + resName + "\": " + result.toString());
                     }
                 }
             }