From 83908b7cfdb0ff57f237a623f4b4e782a75588e2 Mon Sep 17 00:00:00 2001 From: Michael Lipp Date: Fri, 4 Oct 2024 15:01:58 +0000 Subject: [PATCH 1/3] Deploy pod instead of stateful set --- buildSrc/.settings/org.eclipse.jdt.core.prefs | 13 +- .../org.eclipse.jdt.groovy.core.prefs | 2 +- buildSrc/build.gradle | 52 +---- buildSrc/settings.gradle | 7 - deploy/vmop-role.yaml | 2 + org.jdrupes.vmoperator.common/build.gradle | 1 + .../vmoperator/common/K8sObserver.java | 62 +++--- .../vmoperator/common/K8sV1PvcStub.java | 82 ++++++++ .../vmoperator/common/VmDefinitionModel.java | 19 ++ .../build.gradle | 1 - org.jdrupes.vmoperator.manager/build.gradle | 4 +- .../vmoperator/manager/runnerDataPvc.ftl.yaml | 18 ++ .../vmoperator/manager/runnerDiskPvc.ftl.yaml | 16 ++ .../vmoperator/manager/runnerPod.ftl.yaml | 134 ++++++++++++ .../vmoperator/manager/AbstractMonitor.java | 4 +- .../vmoperator/manager/PodReconciler.java | 115 ++++++++++ .../vmoperator/manager/PvcReconciler.java | 197 ++++++++++++++++++ .../vmoperator/manager/Reconciler.java | 30 ++- .../manager/StatefulSetReconciler.java | 49 +++-- .../vmoperator/manager/BasicTests.java | 25 ++- .../build.gradle | 6 +- webpages/vm-operator/controller.md | 59 +++--- webpages/vm-operator/upgrading.md | 19 ++ 23 files changed, 762 insertions(+), 155 deletions(-) delete mode 100644 buildSrc/settings.gradle create mode 100644 org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sV1PvcStub.java create mode 100644 org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml create mode 100644 org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml create mode 100644 org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml create mode 100644 org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java create mode 100644 org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java diff --git a/buildSrc/.settings/org.eclipse.jdt.core.prefs b/buildSrc/.settings/org.eclipse.jdt.core.prefs index 68fda12..b25073a 100644 --- a/buildSrc/.settings/org.eclipse.jdt.core.prefs +++ b/buildSrc/.settings/org.eclipse.jdt.core.prefs @@ -1,9 +1,7 @@ +# +#Wed Oct 02 14:48:43 CEST 2024 eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull -org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable -org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate org.eclipse.jdt.core.compiler.codegen.targetPlatform=21 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve org.eclipse.jdt.core.compiler.compliance=21 @@ -11,12 +9,5 @@ org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error -org.eclipse.jdt.core.compiler.problem.nullReference=warning -org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error -org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=warning -org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=21 diff --git a/buildSrc/.settings/org.eclipse.jdt.groovy.core.prefs b/buildSrc/.settings/org.eclipse.jdt.groovy.core.prefs index bf0ca13..71b5e37 100644 --- a/buildSrc/.settings/org.eclipse.jdt.groovy.core.prefs +++ b/buildSrc/.settings/org.eclipse.jdt.groovy.core.prefs @@ -1,3 +1,3 @@ eclipse.preferences.version=1 -groovy.compiler.level=40 +groovy.compiler.level=-1 groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index a9fb634..4a5db6d 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -1,9 +1,3 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * This project uses @Incubating APIs which are subject to change. - */ - plugins { // Support convention plugins written in Groovy. Convention plugins // are build scripts in 'src/main' that automatically become available @@ -14,52 +8,24 @@ plugins { id 'eclipse' } -repositories { - // Use the plugin portal to apply community plugins in convention plugins. - gradlePluginPortal() -} - sourceSets { - main { - groovy { - srcDirs = ['src'] - } - } - - test { - groovy { - srcDirs = ['test'] - } - } + main { + groovy { + srcDirs = ['src'] + } + resources { + srcDirs = ['resources'] + } + } } eclipse { - project { - file { - // closure executed after .project content is loaded from existing file - // and before gradle build information is merged - beforeMerged { project -> - project.natures.clear() - project.buildCommands.clear() - } - - project.natures += 'org.eclipse.buildship.core.gradleprojectnature' - // Don't build, result not used by Eclipse anyway - // project.buildCommand 'org.eclipse.buildship.core.gradleprojectbuilder' - } - } - - classpath { - downloadJavadoc = true - downloadSources = true - } - jdt { file { withProperties { properties -> def formatterPrefs = new Properties() - rootProject.file("gradle/org.eclipse.jdt.core.formatter.prefs") + rootProject.file("../gradle/org.eclipse.jdt.core.formatter.prefs") .withInputStream { formatterPrefs.load(it) } properties.putAll(formatterPrefs) } diff --git a/buildSrc/settings.gradle b/buildSrc/settings.gradle deleted file mode 100644 index 3f67e42..0000000 --- a/buildSrc/settings.gradle +++ /dev/null @@ -1,7 +0,0 @@ -/* - * This file was generated by the Gradle 'init' task. - * - * This settings file is used to specify which projects to include in your build-logic build. - */ - -rootProject.name = 'buildSrc' diff --git a/deploy/vmop-role.yaml b/deploy/vmop-role.yaml index 0b0e94a..16c7cfb 100644 --- a/deploy/vmop-role.yaml +++ b/deploy/vmop-role.yaml @@ -28,9 +28,11 @@ rules: - apiGroups: - "" resources: + - persistentvolumeclaims - pods verbs: - list - get + - create - delete - patch diff --git a/org.jdrupes.vmoperator.common/build.gradle b/org.jdrupes.vmoperator.common/build.gradle index 42c05ae..07877fe 100644 --- a/org.jdrupes.vmoperator.common/build.gradle +++ b/org.jdrupes.vmoperator.common/build.gradle @@ -10,6 +10,7 @@ plugins { dependencies { api project(':org.jdrupes.vmoperator.util') + api 'org.jgrapes:org.jgrapes.core:[1.22.1,2)' api 'io.kubernetes:client-java:[19.0.0,20.0.0)' api 'org.yaml:snakeyaml' } diff --git a/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sObserver.java b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sObserver.java index 600b5ca..80e3863 100644 --- a/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sObserver.java +++ b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sObserver.java @@ -30,6 +30,7 @@ import java.time.Instant; import java.util.function.BiConsumer; import java.util.logging.Level; import java.util.logging.Logger; +import org.jgrapes.core.Components; /** * An observer that watches namespaced resources in a given context and @@ -73,7 +74,7 @@ public class K8sObserver objectClass, Class objectListClass, K8sClient client, APIResource context, String namespace, ListOptions options) { @@ -85,38 +86,41 @@ public class K8sObserver(objectClass, objectListClass, context.getGroup(), context.getPreferredVersion(), context.getResourcePlural(), client); - thread = Thread.ofVirtual().unstarted(() -> { - try { - logger.config(() -> "Watching " + context.getResourcePlural() - + " (" + context.getPreferredVersion() + ")" - + " in " + namespace); + thread = (Components.useVirtualThreads() ? Thread.ofVirtual() + : Thread.ofPlatform()).unstarted(() -> { + try { + logger + .config(() -> "Watching " + context.getResourcePlural() + + " (" + context.getPreferredVersion() + ")" + + " in " + namespace); - // Watch sometimes terminates without apparent reason. - while (!Thread.currentThread().isInterrupted()) { - Instant startedAt = Instant.now(); - try { - @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") - var changed = api.watch(namespace, options).iterator(); - while (changed.hasNext()) { - handler.accept(client, changed.next()); + // Watch sometimes terminates without apparent reason. + while (!Thread.currentThread().isInterrupted()) { + Instant startedAt = Instant.now(); + try { + @SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops") + var changed + = api.watch(namespace, options).iterator(); + while (changed.hasNext()) { + handler.accept(client, changed.next()); + } + } catch (ApiException | RuntimeException e) { + logger.log(Level.FINE, e, () -> "Problem watching" + + " (will retry): " + e.getMessage()); + delayRestart(startedAt); } - } catch (ApiException | RuntimeException e) { - logger.log(Level.FINE, e, () -> "Problem watching" - + " (will retry): " + e.getMessage()); - delayRestart(startedAt); + } + if (onTerminated != null) { + onTerminated.accept(this, null); + } + } catch (Throwable e) { + logger.log(Level.SEVERE, e, () -> "Probem watching: " + + e.getMessage()); + if (onTerminated != null) { + onTerminated.accept(this, e); } } - if (onTerminated != null) { - onTerminated.accept(this, null); - } - } catch (Throwable e) { - logger.log(Level.SEVERE, e, () -> "Probem watching: " - + e.getMessage()); - if (onTerminated != null) { - onTerminated.accept(this, e); - } - } - }); + }); } @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") diff --git a/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sV1PvcStub.java b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sV1PvcStub.java new file mode 100644 index 0000000..876e648 --- /dev/null +++ b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/K8sV1PvcStub.java @@ -0,0 +1,82 @@ +/* + * VM-Operator + * Copyright (C) 2024 Michael N. Lipp + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.jdrupes.vmoperator.common; + +import io.kubernetes.client.Discovery.APIResource; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim; +import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList; +import io.kubernetes.client.util.generic.options.ListOptions; +import java.util.Collection; +import java.util.List; + +/** + * A stub for pods (v1). + */ +@SuppressWarnings("PMD.DataflowAnomalyAnalysis") +public class K8sV1PvcStub extends + K8sGenericStub { + + /** The pods' context. */ + public static final APIResource CONTEXT + = new APIResource("", List.of("v1"), "v1", "PersistentVolumeClaim", + true, "persistentvolumeclaims", "persistentvolumeclaim"); + + /** + * Instantiates a new stub. + * + * @param client the client + * @param namespace the namespace + * @param name the name + */ + protected K8sV1PvcStub(K8sClient client, String namespace, String name) { + super(V1PersistentVolumeClaim.class, V1PersistentVolumeClaimList.class, + client, CONTEXT, namespace, name); + } + + /** + * Gets the stub for the given namespace and name. + * + * @param client the client + * @param namespace the namespace + * @param name the name + * @return the kpod stub + */ + public static K8sV1PvcStub get(K8sClient client, String namespace, + String name) { + return new K8sV1PvcStub(client, namespace, name); + } + + /** + * Get the stubs for the objects in the given namespace that match + * the criteria from the given options. + * + * @param client the client + * @param namespace the namespace + * @param options the options + * @return the collection + * @throws ApiException the api exception + */ + public static Collection list(K8sClient client, + String namespace, ListOptions options) throws ApiException { + return K8sGenericStub.list(V1PersistentVolumeClaim.class, + V1PersistentVolumeClaimList.class, client, CONTEXT, namespace, + options, (clnt, nscp, name) -> new K8sV1PvcStub(clnt, nscp, name)); + } +} \ No newline at end of file diff --git a/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/VmDefinitionModel.java b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/VmDefinitionModel.java index 5e1ebb0..987b4c8 100644 --- a/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/VmDefinitionModel.java +++ b/org.jdrupes.vmoperator.common/src/org/jdrupes/vmoperator/common/VmDefinitionModel.java @@ -37,6 +37,13 @@ import org.jdrupes.vmoperator.util.GsonPtr; @SuppressWarnings("PMD.DataClass") public class VmDefinitionModel extends K8sDynamicModel { + /** + * The VM state from the VM definition. + */ + public enum RequestedVmState { + STOPPED, RUNNING + } + /** * Permissions for accessing and manipulating the VM. */ @@ -111,6 +118,18 @@ public class VmDefinitionModel extends K8sDynamicModel { .flatMap(Function.identity()).collect(Collectors.toSet()); } + /** + * Return the requested VM state + * + * @return the string + */ + public RequestedVmState vmState() { + return GsonPtr.to(data()).getAsString("spec", "vm", "state") + .map(s -> "Running".equals(s) ? RequestedVmState.RUNNING + : RequestedVmState.STOPPED) + .orElse(RequestedVmState.STOPPED); + } + /** * Get the display password serial. * diff --git a/org.jdrupes.vmoperator.manager.events/build.gradle b/org.jdrupes.vmoperator.manager.events/build.gradle index 2546c6a..cfdd79e 100644 --- a/org.jdrupes.vmoperator.manager.events/build.gradle +++ b/org.jdrupes.vmoperator.manager.events/build.gradle @@ -9,7 +9,6 @@ plugins { } dependencies { - api 'org.jgrapes:org.jgrapes.core:[1.21.0,2)' api project(':org.jdrupes.vmoperator.common') api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[2.16.1,3]' } diff --git a/org.jdrupes.vmoperator.manager/build.gradle b/org.jdrupes.vmoperator.manager/build.gradle index 4c98978..a4c302e 100644 --- a/org.jdrupes.vmoperator.manager/build.gradle +++ b/org.jdrupes.vmoperator.manager/build.gradle @@ -13,8 +13,8 @@ dependencies { implementation 'commons-cli:commons-cli:1.5.0' - implementation 'org.jgrapes:org.jgrapes.util:[1.36.0,2)' - implementation 'org.jgrapes:org.jgrapes.io:[2.11.0,3)' + implementation 'org.jgrapes:org.jgrapes.util:[1.38.1,2)' + implementation 'org.jgrapes:org.jgrapes.io:[2.12.1,3)' implementation 'org.jgrapes:org.jgrapes.http:[3.5.0,4)' implementation 'org.jgrapes:org.jgrapes.webconsole.base:[1.8.0,2)' diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml new file mode 100644 index 0000000..a1a94e1 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml @@ -0,0 +1,18 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + namespace: ${ cr.metadata.namespace.asString } + name: ${ runnerDataPvcName } + labels: + app.kubernetes.io/name: ${ constants.APP_NAME } + app.kubernetes.io/instance: ${ cr.metadata.name.asString } + app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME } +spec: + accessModes: + - ReadWriteOnce + <#if reconciler.runnerDataPvc?? && reconciler.runnerDataPvc.storageClassName??> + storageClassName: ${ reconciler.runnerDataPvc.storageClassName } + + resources: + requests: + storage: 1Mi diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml new file mode 100644 index 0000000..732f592 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml @@ -0,0 +1,16 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + namespace: ${ cr.metadata.namespace.asString } + name: ${ disk.generatedPvcName } + labels: + app.kubernetes.io/name: ${ constants.APP_NAME } + app.kubernetes.io/instance: ${ cr.metadata.name.asString } + app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME } + <#if disk.volumeClaimTemplate.metadata?? + && disk.volumeClaimTemplate.metadata.annotations??> + annotations: + ${ disk.volumeClaimTemplate.metadata.annotations.toString() } + +spec: + ${ disk.volumeClaimTemplate.spec.toString() } diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml new file mode 100644 index 0000000..ad652e4 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml @@ -0,0 +1,134 @@ +kind: Pod +apiVersion: v1 +metadata: + namespace: ${ cr.metadata.namespace.asString } + name: ${ cr.metadata.name.asString } + labels: + app.kubernetes.io/name: ${ constants.APP_NAME } + app.kubernetes.io/instance: ${ cr.metadata.name.asString } + app.kubernetes.io/component: ${ constants.APP_NAME } + app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME } + annotations: + # Triggers update of config map mounted in pod + # See https://ahmet.im/blog/kubernetes-secret-volumes-delay/ + vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }" + vmoperator.jdrupes.org/version: ${ managerVersion } + ownerReferences: + - apiVersion: ${ cr.apiVersion.asString } + kind: ${ constants.VM_OP_KIND_VM } + name: ${ cr.metadata.name.asString } + uid: ${ cr.metadata.uid.asString } + blockOwnerDeletion: true + controller: false +spec: + containers: + - name: ${ cr.metadata.name.asString } + <#assign image = cr.spec.image> + <#if image.source??> + image: ${ image.source.asString } + <#else> + image: ${ image.repository.asString }/${ image.path.asString }<#if image.version??>:${ image.version.asString } + + <#if image.pullPolicy??> + imagePullPolicy: ${ image.pullPolicy.asString } + + <#if cr.spec.vm.display.spice??> + ports: + <#if cr.spec.vm.display.spice??> + - name: spice + containerPort: ${ cr.spec.vm.display.spice.port.asInt?c } + protocol: TCP + + + volumeMounts: + # Not needed because pod is priviledged: + # - mountPath: /dev/kvm + # name: dev-kvm + # - mountPath: /dev/net/tun + # name: dev-tun + # - mountPath: /sys/fs/cgroup + # name: cgroup + - name: config + mountPath: /etc/opt/vmrunner + - name: runner-data + mountPath: /var/local/vm-data + - name: vmop-image-repository + mountPath: ${ constants.IMAGE_REPO_PATH } + volumeDevices: + <#list cr.spec.vm.disks.asList() as disk> + <#if disk.volumeClaimTemplate??> + - name: ${ disk.generatedDiskName.asString } + devicePath: /dev/${ disk.generatedDiskName.asString } + + + securityContext: + privileged: true + <#if cr.spec.resources??> + resources: ${ cr.spec.resources.toString() } + <#else> + <#if cr.spec.vm.currentCpus?? || cr.spec.vm.currentRam?? > + resources: + requests: + <#if cr.spec.vm.currentCpus?? > + <#assign factor = 2.0 /> + <#if reconciler.cpuOvercommit??> + <#assign factor = reconciler.cpuOvercommit * 1.0 /> + + cpu: ${ (parseQuantity(cr.spec.vm.currentCpus.asString) / factor)?c } + + <#if cr.spec.vm.currentRam?? > + <#assign factor = 1.25 /> + <#if reconciler.ramOvercommit??> + <#assign factor = reconciler.ramOvercommit * 1.0 /> + + memory: ${ (parseQuantity(cr.spec.vm.currentRam.asString) / factor)?floor?c } + + + + volumes: + # Not needed because pod is priviledged: + # - name: dev-kvm + # hostPath: + # path: /dev/kvm + # type: CharDevice + # - hostPath: + # path: /dev/net/tun + # type: CharDevice + # name: dev-tun + # - name: cgroup + # hostPath: + # path: /sys/fs/cgroup + - name: config + projected: + sources: + - configMap: + name: ${ cr.metadata.name.asString } + <#if displaySecret??> + - secret: + name: ${ displaySecret } + + - name: vmop-image-repository + persistentVolumeClaim: + claimName: vmop-image-repository + - name: runner-data + persistentVolumeClaim: + claimName: ${ runnerDataPvcName } + <#list cr.spec.vm.disks.asList() as disk> + <#if disk.volumeClaimTemplate??> + - name: ${ disk.generatedDiskName.asString } + persistentVolumeClaim: + claimName: ${ disk.generatedPvcName.asString } + + + hostNetwork: true + terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c } + <#if cr.spec.nodeName??> + nodeName: ${ cr.spec.nodeName.asString } + + <#if cr.spec.nodeSelector??> + nodeSelector: ${ cr.spec.nodeSelector.toString() } + + <#if cr.spec.affinity??> + affinity: ${ cr.spec.affinity.toString() } + + serviceAccountName: vm-runner diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java index e78a5e0..0c9b08c 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java @@ -62,7 +62,6 @@ public abstract class AbstractMonitor channelManager; - private boolean channelManagerMaster; /** * Initializes the instance. @@ -240,8 +239,7 @@ public abstract class AbstractMonitor { handleChange(c, r); - if (ResponseType.valueOf(r.type) == ResponseType.DELETED - && channelManagerMaster) { + if (ResponseType.valueOf(r.type) == ResponseType.DELETED) { channelManager.remove(r.object.getMetadata().getName()); } }).onTerminated((o, t) -> { diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java new file mode 100644 index 0000000..33c2221 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java @@ -0,0 +1,115 @@ +/* + * VM-Operator + * Copyright (C) 2023 Michael N. Lipp + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.jdrupes.vmoperator.manager; + +import freemarker.template.Configuration; +import freemarker.template.TemplateException; +import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.util.generic.dynamic.Dynamics; +import io.kubernetes.client.util.generic.options.PatchOptions; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Map; +import java.util.logging.Logger; +import org.jdrupes.vmoperator.common.K8sV1PodStub; +import org.jdrupes.vmoperator.common.VmDefinitionModel.RequestedVmState; +import org.jdrupes.vmoperator.manager.events.VmChannel; +import org.jdrupes.vmoperator.manager.events.VmDefChanged; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +/** + * Delegee for reconciling the pod. + */ +@SuppressWarnings("PMD.DataflowAnomalyAnalysis") +/* default */ class PodReconciler { + + protected final Logger logger = Logger.getLogger(getClass().getName()); + private final Configuration fmConfig; + + /** + * Instantiates a new pod reconciler. + * + * @param fmConfig the fm config + */ + public PodReconciler(Configuration fmConfig) { + this.fmConfig = fmConfig; + } + + /** + * Reconcile the pod. + * + * @param event the event + * @param model the model + * @param channel the channel + * @throws IOException Signals that an I/O exception has occurred. + * @throws TemplateException the template exception + * @throws ApiException the api exception + */ + public void reconcile(VmDefChanged event, Map model, + VmChannel channel) + throws IOException, TemplateException, ApiException { + // Don't do anything if stateful set is still in use (pre v3.4) + if ((Boolean) model.get("usingSts")) { + return; + } + + // Get pod stub. + var metadata = event.vmDefinition().getMetadata(); + var podStub = K8sV1PodStub.get(channel.client(), + metadata.getNamespace(), metadata.getName()); + + // Nothing to do if exists and should be running + if (event.vmDefinition().vmState() == RequestedVmState.RUNNING + && podStub.model().isPresent()) { + return; + } + + // Delete if running but should be stopped + if (event.vmDefinition().vmState() == RequestedVmState.STOPPED) { + if (podStub.model().isPresent()) { + podStub.delete(); + } + return; + } + + // Create pod. First combine template and data and parse result + var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml"); + StringWriter out = new StringWriter(); + fmTemplate.process(model, out); + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var podDef = Dynamics.newFromYaml( + new Yaml(new SafeConstructor(new LoaderOptions())), out.toString()); + + // Do apply changes + PatchOptions opts = new PatchOptions(); + opts.setForce(true); + opts.setFieldManager("kubernetes-java-kubectl-apply"); + if (podStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML, + new V1Patch(channel.client().getJSON().serialize(podDef)), opts) + .isEmpty()) { + logger.warning( + () -> "Could not patch pod for " + podStub.name()); + } + } + +} diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java new file mode 100644 index 0000000..39fcfe9 --- /dev/null +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java @@ -0,0 +1,197 @@ +/* + * VM-Operator + * Copyright (C) 2023 Michael N. Lipp + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.jdrupes.vmoperator.manager; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import freemarker.core.ParseException; +import freemarker.template.Configuration; +import freemarker.template.MalformedTemplateNameException; +import freemarker.template.TemplateException; +import freemarker.template.TemplateNotFoundException; +import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.util.generic.dynamic.Dynamics; +import io.kubernetes.client.util.generic.options.ListOptions; +import io.kubernetes.client.util.generic.options.PatchOptions; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import static org.jdrupes.vmoperator.common.Constants.APP_NAME; +import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME; +import org.jdrupes.vmoperator.common.K8sV1PvcStub; +import org.jdrupes.vmoperator.manager.events.VmChannel; +import org.jdrupes.vmoperator.manager.events.VmDefChanged; +import org.jdrupes.vmoperator.util.GsonPtr; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +/** + * Delegee for reconciling the stateful set (effectively the pod). + */ +@SuppressWarnings("PMD.DataflowAnomalyAnalysis") +/* default */ class PvcReconciler { + + protected final Logger logger = Logger.getLogger(getClass().getName()); + private final Configuration fmConfig; + + /** + * Instantiates a new pvc reconciler. + * + * @param fmConfig the fm config + */ + public PvcReconciler(Configuration fmConfig) { + this.fmConfig = fmConfig; + } + + /** + * Reconcile the PVCs. + * + * @param event the event + * @param model the model + * @param channel the channel + * @throws IOException Signals that an I/O exception has occurred. + * @throws TemplateException the template exception + * @throws ApiException the api exception + */ + @SuppressWarnings("PMD.AvoidDuplicateLiterals") + public void reconcile(VmDefChanged event, Map model, + VmChannel channel) + throws IOException, TemplateException, ApiException { + var metadata = event.vmDefinition().getMetadata(); + + // Existing disks + ListOptions listOpts = new ListOptions(); + listOpts.setLabelSelector( + "app.kubernetes.io/managed-by=" + VM_OP_NAME + "," + + "app.kubernetes.io/name=" + APP_NAME + "," + + "app.kubernetes.io/instance=" + metadata.getName()); + var knownDisks = K8sV1PvcStub.list(channel.client(), + metadata.getNamespace(), listOpts); + var knownPvcs = knownDisks.stream().map(K8sV1PvcStub::name) + .collect(Collectors.toSet()); + + // Reconcile runner data pvc + reconcileRunnerDataPvc(event, model, channel, knownPvcs); + + // Reconcile pvcs for defined disks + var diskDefs = GsonPtr.to((JsonObject) model.get("cr")) + .getAsListOf(JsonObject.class, "spec", "vm", "disks"); + var diskCounter = 0; + for (var diskDef : diskDefs) { + if (!diskDef.has("volumeClaimTemplate")) { + continue; + } + var diskName = GsonPtr.to(diskDef) + .getAsString("volumeClaimTemplate", "metadata", "name") + .map(name -> name + "-disk").orElse("disk-" + diskCounter); + diskCounter += 1; + diskDef.addProperty("generatedDiskName", diskName); + + // Don't do anything if pvc with old (sts generated) name exists. + var stsDiskPvcName = diskName + "-" + metadata.getName() + "-0"; + if (knownPvcs.contains(stsDiskPvcName)) { + diskDef.addProperty("generatedPvcName", stsDiskPvcName); + continue; + } + + // Update PVC + model.put("disk", diskDef); + reconcileRunnerDiskPvc(event, model, channel); + } + model.remove("disk"); + } + + private void reconcileRunnerDataPvc(VmDefChanged event, + Map model, VmChannel channel, + Set knownPvcs) + throws TemplateNotFoundException, MalformedTemplateNameException, + ParseException, IOException, TemplateException, ApiException { + var metadata = event.vmDefinition().getMetadata(); + + // Look for old (sts generated) name. + var stsRunnerDataPvcName + = "runner-data" + "-" + metadata.getName() + "-0"; + if (knownPvcs.contains(stsRunnerDataPvcName)) { + model.put("runnerDataPvcName", stsRunnerDataPvcName); + return; + } + + // Generate PVC + model.put("runnerDataPvcName", metadata.getName() + "-runner-data"); + var fmTemplate = fmConfig.getTemplate("runnerDataPvc.ftl.yaml"); + StringWriter out = new StringWriter(); + fmTemplate.process(model, out); + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var pvcDef = Dynamics.newFromYaml( + new Yaml(new SafeConstructor(new LoaderOptions())), out.toString()); + + // Do apply changes + var pvcStub = K8sV1PvcStub.get(channel.client(), + metadata.getNamespace(), (String) model.get("runnerDataPvcName")); + PatchOptions opts = new PatchOptions(); + opts.setForce(true); + opts.setFieldManager("kubernetes-java-kubectl-apply"); + if (pvcStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML, + new V1Patch(channel.client().getJSON().serialize(pvcDef)), opts) + .isEmpty()) { + logger.warning( + () -> "Could not patch pvc for " + pvcStub.name()); + } + } + + private void reconcileRunnerDiskPvc(VmDefChanged event, + Map model, VmChannel channel) + throws TemplateNotFoundException, MalformedTemplateNameException, + ParseException, IOException, TemplateException, ApiException { + var metadata = event.vmDefinition().getMetadata(); + + // Generate PVC + var diskDef = GsonPtr.to((JsonElement) model.get("disk")); + var pvcName = metadata.getName() + "-" + + diskDef.getAsString("generatedDiskName").get(); + diskDef.set("generatedPvcName", pvcName); + var fmTemplate = fmConfig.getTemplate("runnerDiskPvc.ftl.yaml"); + StringWriter out = new StringWriter(); + fmTemplate.process(model, out); + // Avoid Yaml.load due to + // https://github.com/kubernetes-client/java/issues/2741 + var pvcDef = Dynamics.newFromYaml( + new Yaml(new SafeConstructor(new LoaderOptions())), out.toString()); + + // Do apply changes + var pvcStub = K8sV1PvcStub.get(channel.client(), + metadata.getNamespace(), GsonPtr.to((JsonElement) model.get("disk")) + .getAsString("generatedPvcName").get()); + PatchOptions opts = new PatchOptions(); + opts.setForce(true); + opts.setFieldManager("kubernetes-java-kubectl-apply"); + if (pvcStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML, + new V1Patch(channel.client().getJSON().serialize(pvcDef)), opts) + .isEmpty()) { + logger.warning( + () -> "Could not patch pvc for " + pvcStub.name()); + } + } +} diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java index deb8680..17ba60d 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java @@ -69,20 +69,25 @@ import org.jgrapes.util.events.ConfigurationUpdate; * * * A [`ConfigMap`](https://kubernetes.io/docs/concepts/configuration/configmap/) * that defines the configuration file for the runner. - * - * * A [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) - * that creates - * * the [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) - * with the Runner instance, - * * a PVC for 1 MiB of persistent storage used by the Runner - * (referred to as the "runnerDataPvc") and - * * the PVCs for the VM's disks. - * + * + * * A [`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) + * for 1 MiB of persistent storage used by the Runner (referred to as the + * "runnerDataPvc") + * + * * The PVCs for the VM's disks. + * + * * A [`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) with the + * runner instance[^oldSts]. + * * * (Optional) A load balancer * [`Service`](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/) * that allows the user to access a VM's console without knowing which * node it runs on. * + * [^oldSts]: Before version 3.4, the operator created a + * [`StatefulSet`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + * that created the pod. + * * The reconciler is part of the {@link Controller} component. It's * configuration properties are therefore defined in * ```yaml @@ -135,6 +140,8 @@ public class Reconciler extends Component { private final ConfigMapReconciler cmReconciler; private final DisplaySecretReconciler dsReconciler; private final StatefulSetReconciler stsReconciler; + private final PvcReconciler pvcReconciler; + private final PodReconciler podReconciler; private final LoadBalancerReconciler lbReconciler; @SuppressWarnings("PMD.UseConcurrentHashMap") private final Map config = new HashMap<>(); @@ -160,6 +167,8 @@ public class Reconciler extends Component { cmReconciler = new ConfigMapReconciler(fmConfig); dsReconciler = new DisplaySecretReconciler(); stsReconciler = new StatefulSetReconciler(fmConfig); + pvcReconciler = new PvcReconciler(fmConfig); + podReconciler = new PodReconciler(fmConfig); lbReconciler = new LoadBalancerReconciler(fmConfig); } @@ -206,7 +215,10 @@ public class Reconciler extends Component { var configMap = cmReconciler.reconcile(model, channel); model.put("cm", configMap.getRaw()); dsReconciler.reconcile(event, model, channel); + // Manage (eventual) removal of stateful set. stsReconciler.reconcile(event, model, channel); + pvcReconciler.reconcile(event, model, channel); + podReconciler.reconcile(event, model, channel); lbReconciler.reconcile(event, model, channel); } diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java index baf833c..a58d169 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java @@ -37,7 +37,9 @@ import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; /** - * Delegee for reconciling the stateful set (effectively the pod). + * Before version 3.4, the pod running the VM was created by a stateful set. + * Starting with version 3.4, this reconciler simply deletes the stateful + * set, provided that the VM is not running. */ @SuppressWarnings("PMD.DataflowAnomalyAnalysis") /* default */ class StatefulSetReconciler { @@ -46,7 +48,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; private final Configuration fmConfig; /** - * Instantiates a new config map reconciler. + * Instantiates a new stateful set reconciler. * * @param fmConfig the fm config */ @@ -64,12 +66,34 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; * @throws TemplateException the template exception * @throws ApiException the api exception */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public void reconcile(VmDefChanged event, Map model, VmChannel channel) throws IOException, TemplateException, ApiException { var metadata = event.vmDefinition().getMetadata(); + model.put("usingSts", false); - // Combine template and data and parse result + // If exists, delete when not running or supposed to be not running. + var stsStub = K8sV1StatefulSetStub.get(channel.client(), + metadata.getNamespace(), metadata.getName()); + if (stsStub.model().isEmpty()) { + return; + } + + // Stateful set still exists, check if replicas is 0 so we can + // delete it. + var stsModel = stsStub.model().get(); + if (stsModel.getSpec().getReplicas() == 0) { + stsStub.delete(); + return; + } + + // Cannot yet delete the stateful set. + model.put("usingSts", true); + + // Check if VM is supposed to be stopped. If so, + // set replicas to 0. This is the first step of the transition, + // the stateful set will be deleted when the VM is restarted. var fmTemplate = fmConfig.getTemplate("runnerSts.ftl.yaml"); StringWriter out = new StringWriter(); fmTemplate.process(model, out); @@ -77,22 +101,13 @@ import org.yaml.snakeyaml.constructor.SafeConstructor; // https://github.com/kubernetes-client/java/issues/2741 var stsDef = Dynamics.newFromYaml( new Yaml(new SafeConstructor(new LoaderOptions())), out.toString()); - - // If exists apply changes only when transitioning state - // or not running. - var stsStub = K8sV1StatefulSetStub.get(channel.client(), - metadata.getNamespace(), metadata.getName()); - var stsModel = stsStub.model().orElse(null); - if (stsModel != null) { - var current = stsModel.getSpec().getReplicas(); - var desired = GsonPtr.to(stsDef.getRaw()) - .to("spec").getAsInt("replicas").orElse(1); - if (current == 1 && desired == 1) { - return; - } + var desired = GsonPtr.to(stsDef.getRaw()) + .to("spec").getAsInt("replicas").orElse(1); + if (desired == 1) { + return; } - // Do apply changes + // Do apply changes (set replicas to 0) PatchOptions opts = new PatchOptions(); opts.setForce(true); opts.setFieldManager("kubernetes-java-kubectl-apply"); diff --git a/org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java b/org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java index 13a93e1..bd5fb54 100644 --- a/org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java +++ b/org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java @@ -2,17 +2,24 @@ package org.jdrupes.vmoperator.manager; import io.kubernetes.client.Discovery.APIResource; import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.util.generic.options.ListOptions; + import java.io.FileReader; import java.io.IOException; import java.util.Map; + +import static org.jdrupes.vmoperator.common.Constants.APP_NAME; import static org.jdrupes.vmoperator.common.Constants.VM_OP_GROUP; import static org.jdrupes.vmoperator.common.Constants.VM_OP_KIND_VM; +import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME; + import org.jdrupes.vmoperator.common.K8s; import org.jdrupes.vmoperator.common.K8sClient; import org.jdrupes.vmoperator.common.K8sDynamicStub; import org.jdrupes.vmoperator.common.K8sV1ConfigMapStub; import org.jdrupes.vmoperator.common.K8sV1DeploymentStub; -import org.jdrupes.vmoperator.common.K8sV1StatefulSetStub; +import org.jdrupes.vmoperator.common.K8sV1PodStub; +import org.jdrupes.vmoperator.common.K8sV1PvcStub; import org.junit.jupiter.api.AfterAll; import static org.junit.jupiter.api.Assertions.*; import org.junit.jupiter.api.BeforeAll; @@ -78,7 +85,7 @@ class BasicTests { // Wait for created resources assertTrue(waitForConfigMap(client)); - assertTrue(waitForStatefulSet(client)); + assertTrue(waitForPvc(client)); // Check config map var config = K8sV1ConfigMapStub.get(client, "vmop-dev", "unittest-vm") @@ -93,6 +100,15 @@ class BasicTests { // Cleanup K8sDynamicStub.get(client, vmsContext, "vmop-dev", "unittest-vm") .delete(); + ListOptions listOpts = new ListOptions(); + listOpts.setLabelSelector( + "app.kubernetes.io/managed-by=" + VM_OP_NAME + "," + + "app.kubernetes.io/name=" + APP_NAME + "," + + "app.kubernetes.io/instance=unittest-vm"); + var knownPvcs = K8sV1PvcStub.list(client, "vmop-dev", listOpts); + for (var pvc : knownPvcs) { + pvc.delete(); + } } private boolean waitForConfigMap(K8sClient client) @@ -107,9 +123,10 @@ class BasicTests { return false; } - private boolean waitForStatefulSet(K8sClient client) + private boolean waitForPvc(K8sClient client) throws InterruptedException, ApiException { - var stub = K8sV1StatefulSetStub.get(client, "vmop-dev", "unittest-vm"); + var stub + = K8sV1PvcStub.get(client, "vmop-dev", "unittest-vm-runner-data"); for (int i = 0; i < 10; i++) { if (stub.model().isPresent()) { return true; diff --git a/org.jdrupes.vmoperator.runner.qemu/build.gradle b/org.jdrupes.vmoperator.runner.qemu/build.gradle index 7a0e2bd..695c815 100644 --- a/org.jdrupes.vmoperator.runner.qemu/build.gradle +++ b/org.jdrupes.vmoperator.runner.qemu/build.gradle @@ -9,9 +9,9 @@ plugins { } dependencies { - implementation 'org.jgrapes:org.jgrapes.core:[1.21.0,2)' - implementation 'org.jgrapes:org.jgrapes.util:[1.36.0,2)' - implementation 'org.jgrapes:org.jgrapes.io:[2.11.0,3)' + implementation 'org.jgrapes:org.jgrapes.core:[1.22.1,2)' + implementation 'org.jgrapes:org.jgrapes.util:[1.38.1,2)' + implementation 'org.jgrapes:org.jgrapes.io:[2.12.1,3)' implementation 'org.jgrapes:org.jgrapes.http:[3.5.0,4)' implementation project(':org.jdrupes.vmoperator.common') diff --git a/webpages/vm-operator/controller.md b/webpages/vm-operator/controller.md index a24943d..17ca790 100644 --- a/webpages/vm-operator/controller.md +++ b/webpages/vm-operator/controller.md @@ -61,16 +61,20 @@ spec: ## Pod management The central resource created by the controller is a -[stateful set](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) -with the same name as the VM (metadata.name). Its number of replicas is -set to 1 if `spec.vm.state` is "Running" (default is "Stopped" which sets -replicas to 0). +[`Pod`](https://kubernetes.io/docs/concepts/workloads/pods/) +with the same name as the VM (`metadata.name`). The pod is created only +if `spec.vm.state` is "Running" (default is "Stopped" which deletes the +pod)[^oldSts]. Property `spec.guestShutdownStops` (since 2.2.0) controls the effect of a -shutdown initiated by the guest. If set to `false` (default) a new pod -is automatically created by the stateful set controller and the VM thus -restarted. If set to `true`, the runner sets `spec.vm.state` to "Stopped" -before terminating and by this prevents the creation of a new pod. +shutdown initiated by the guest. If set to `false` (default) the pod +and thus the VM is automatically restarted. If set to `true`, the +VM's state is set to "Stopped" when the VM terminates and the pod is +deleted. + +[^oldSts]: Before version 3.4, the operator created a + [stateful set](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) + that in turn created the pod and the PVCs (see below). ## Defining the basics @@ -84,7 +88,8 @@ running VMs. Maybe the most interesting part is the definition of the VM's disks. This is done by adding one or more `volumeClaimTemplate`s to the list of disks. As its name suggests, such a template is used by the -controller to generate a PVC. +controller to generate a +[`PVC`](https://kubernetes.io/docs/concepts/storage/persistent-volumes/). The example template does not define any storage. Rather it references some PV that you must have created first. This may be your first approach @@ -110,24 +115,28 @@ as shown in this example: The disk will be available as "/dev/*name*-disk" in the VM, using the string from `.volumeClaimTemplate.metadata.name` as *name*. If no name is defined in the metadata, then "/dev/disk-*n*" -is used instead, with *n* being the index of the disk -definition in the list of disks. +is used instead, with *n* being the index of the volume claim +template in the list of disks. -Apart from appending "-disk" to the name (or generating the name) the -`volumeClaimTemplate` is simply copied into the stateful set definition -for the VM (with some additional labels, see below). The controller -for stateful sets appends the started pod's name to the name of the -volume claim templates when it creates the PVCs. Therefore you'll -eventually find the PVCs as "*name*-disk-*vmName*-0" -(or "disk-*n*-*vmName*-0"). +The name of the generated PVC is the VM's name with "-*name*-disk" +(or the generated name) appended: "*vmName*-*name*-disk" +(or "*vmName*-disk-*n*"). The definition of the PVC is simply a copy +of the information from the `volumeClaimTemplate` (with some additional +labels, see below)[^oldStsDisks]. -PVCs generated from stateful set definitions are considered "precious" -and never removed automatically. This behavior fits perfectly for VMs. -Usually, you do not want the disks to be removed automatically when -you (maybe accidentally) remove the CR for the VM. To simplify the lookup -for an eventual (manual) removal, all PVCs are labeled with -"app.kubernetes.io/name: vm-runner", "app.kubernetes.io/instance: *vmName*", -and "app.kubernetes.io/managed-by: vm-operator". +[^oldStsDisks]: Before version 3.4 the `volumeClaimTemplate`s were + copied in the definition of the stateful set. As a stateful set + appends the started pod's name to the name of the volume claim + templates when it creates the PVCs, the PVCs' name were + "*name*-disk-*vmName*-0" (or "disk-*n*-*vmName*-0"). + +PVCs are never removed automatically. Usually, you do not want your +VMs disks to be removed when you (maybe accidentally) remove the CR +for the VM. To simplify the lookup for an eventual (manual) removal, +all PVCs are labeled with "app.kubernetes.io/name: vm-runner", +"app.kubernetes.io/instance: *vmName*", and +"app.kubernetes.io/managed-by: vm-operator", making it easy to select +the PVCs by label in a delete command. ## Choosing an image for the runner diff --git a/webpages/vm-operator/upgrading.md b/webpages/vm-operator/upgrading.md index 7b69716..ec5da7e 100644 --- a/webpages/vm-operator/upgrading.md +++ b/webpages/vm-operator/upgrading.md @@ -5,6 +5,25 @@ layout: vm-operator # Upgrading +## To version 3.4.0 + +Starting with this version, the VM-Operator no longer uses a stateful set +with replica set to 1 to (indirectly) start the pod with the VM. Rather +it creates the pod directly. This implies that the PVCs must also be created +by the VM-Operator, which needs additional permissions to do so (update of +`deploy/vmop-role.yaml). As it would be ridiculous to keep the naming scheme +used by the stateful set when generating PVCs, the VM-Operator uses a +[different pattern](controller.html#defining-disks) for creating new PVCs. + +The change is backward compatible: + + * Running pods created by a stateful set are left alone until stopped. + Only then will the stateful set be removed. + + * The VM-Operator looks for existing PVCs generated by a stateful + set in the pre 3.4 versions (naming pattern "*name*-disk-*vmName*-0") + and reuses them. Only new PVCs are generated using the new pattern. + ## To version 3.0.0 All configuration files are backward compatible to version 2.3.0. From 54445ef5316e8ecbf6a0ca8eac0757bb5da5b846 Mon Sep 17 00:00:00 2001 From: Michael Lipp Date: Sun, 6 Oct 2024 10:05:09 +0000 Subject: [PATCH 2/3] Prepare release v3.4 --- .../manager/events/ChannelDictionary.java | 113 ++++++++++ .../manager/events/ChannelManager.java | 211 +++--------------- ...{ChannelCache.java => ChannelTracker.java} | 148 +++++------- .../manager/events/ServiceChanged.java | 76 ------- .../vmoperator/manager/AbstractMonitor.java | 56 ++--- .../vmoperator/manager/Controller.java | 5 +- .../manager/DisplaySecretMonitor.java | 10 +- .../vmoperator/manager/ServiceMonitor.java | 74 ------ .../jdrupes/vmoperator/manager/VmMonitor.java | 35 ++- .../vmoperator/vmconlet/TimeSeries.java | 4 +- .../jdrupes/vmoperator/vmconlet/VmConlet.java | 20 +- .../jdrupes/vmoperator/vmviewer/VmViewer.java | 26 +-- 12 files changed, 274 insertions(+), 504 deletions(-) create mode 100644 org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelDictionary.java rename org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/{ChannelCache.java => ChannelTracker.java} (52%) delete mode 100644 org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ServiceChanged.java delete mode 100644 org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ServiceMonitor.java diff --git a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelDictionary.java b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelDictionary.java new file mode 100644 index 0000000..05a079e --- /dev/null +++ b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelDictionary.java @@ -0,0 +1,113 @@ +/* + * VM-Operator + * Copyright (C) 2024 Michael N. Lipp + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package org.jdrupes.vmoperator.manager.events; + +import java.util.Collection; +import java.util.Optional; +import java.util.Set; +import org.jgrapes.core.Channel; + +/** + * Supports the lookup of a channel by a name (an id). As a convenience, + * it is possible to additionally associate arbitrary data with the entry + * (and thus with the channel). Note that this interface defines a + * read-only view of the dictionary. + * + * @param the key type + * @param the channel type + * @param the type of the associated data + */ +public interface ChannelDictionary { + + /** + * Combines the channel and the associated data. + * + * @param the channel type + * @param the type of the associated data + * @param channel the channel + * @param associated the associated + */ + @SuppressWarnings("PMD.ShortClassName") + public record Value(C channel, A associated) { + } + + /** + * Returns all known keys. + * + * @return the keys + */ + Set keys(); + + /** + * Return all known values. + * + * @return the collection + */ + Collection> values(); + + /** + * Returns the channel and associates data registered for the key + * or an empty optional if no entry exists. + * + * @param key the key + * @return the result + */ + Optional> value(K key); + + /** + * Return all known channels. + * + * @return the collection + */ + default Collection channels() { + return values().stream().map(v -> v.channel).toList(); + } + + /** + * Returns the channel registered for the key or an empty optional + * if no mapping exists. + * + * @param key the key + * @return the optional + */ + default Optional channel(K key) { + return value(key).map(b -> b.channel); + } + + /** + * Returns all known associated data. + * + * @return the collection + */ + default Collection associated() { + return values().stream() + .filter(v -> v.associated() != null) + .map(v -> v.associated).toList(); + } + + /** + * Return the data associated with the entry for the channel. + * + * @param key the key + * @return the data + */ + default Optional associated(K key) { + return value(key).map(b -> b.associated); + } +} diff --git a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelManager.java b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelManager.java index eb27ea0..2cf7a85 100644 --- a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelManager.java +++ b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelManager.java @@ -27,53 +27,24 @@ import java.util.function.Function; import org.jgrapes.core.Channel; /** - * A channel manager that maintains mappings from a key to a channel. - * As a convenience, it is possible to additionally associate arbitrary - * data with the entry (and thus with the channel). + * Provides an actively managed implementation of the {@link ChannelDictionary}. * - * The manager should be used by a component that defines channels for - * housekeeping. It can be shared between this component and another - * component, preferably using the {@link #fixed()} view for the - * second component. Alternatively, the second component can use a - * {@link ChannelCache} to track the mappings using events. + * The {@link ChannelManager} can be used for housekeeping by any component + * that creates channels. It can be shared between this component and + * some other component, preferably passing it as {@link ChannelDictionary} + * (the read-only view) to the second component. Alternatively, the other + * component can use a {@link ChannelTracker} to track the mappings using + * events. * * @param the key type * @param the channel type * @param the type of the associated data */ -public class ChannelManager { +public class ChannelManager + implements ChannelDictionary { - private final Map> channels = new ConcurrentHashMap<>(); + private final Map> entries = new ConcurrentHashMap<>(); private final Function supplier; - private ChannelManager readOnly; - - /** - * Combines the channel and the associated data. - * - * @param the generic type - * @param the generic type - */ - @SuppressWarnings("PMD.ShortClassName") - public static class Both { - - /** The channel. */ - public C channel; - - /** The associated. */ - public A associated; - - /** - * Instantiates a new both. - * - * @param channel the channel - * @param associated the associated - */ - public Both(C channel, A associated) { - super(); - this.channel = channel; - this.associated = associated; - } - } /** * Instantiates a new channel manager. @@ -91,6 +62,21 @@ public class ChannelManager { this(k -> null); } + @Override + public Set keys() { + return entries.keySet(); + } + + /** + * Return all known values. + * + * @return the collection + */ + @Override + public Collection> values() { + return entries.values(); + } + /** * Returns the channel and associates data registered for the key * or an empty optional if no mapping exists. @@ -98,10 +84,8 @@ public class ChannelManager { * @param key the key * @return the result */ - public Optional> both(K key) { - synchronized (channels) { - return Optional.ofNullable(channels.get(key)); - } + public Optional> value(K key) { + return Optional.ofNullable(entries.get(key)); } /** @@ -113,7 +97,7 @@ public class ChannelManager { * @return the channel manager */ public ChannelManager put(K key, C channel, A associated) { - channels.put(key, new Both<>(channel, associated)); + entries.put(key, new Value<>(channel, associated)); return this; } @@ -129,17 +113,6 @@ public class ChannelManager { return this; } - /** - * Returns the channel registered for the key or an empty optional - * if no mapping exists. - * - * @param key the key - * @return the optional - */ - public Optional channel(K key) { - return both(key).map(b -> b.channel); - } - /** * Returns the {@link Channel} for the given name, creating it using * the supplier passed to the constructor if it doesn't exist yet. @@ -147,8 +120,8 @@ public class ChannelManager { * @param key the key * @return the channel */ - public Optional getChannel(K key) { - return getChannel(key, supplier); + public C channelGet(K key) { + return computeIfAbsent(key, supplier); } /** @@ -161,17 +134,9 @@ public class ChannelManager { */ @SuppressWarnings({ "PMD.AssignmentInOperand", "PMD.DataflowAnomalyAnalysis" }) - public Optional getChannel(K key, Function supplier) { - synchronized (channels) { - return Optional - .of(Optional.ofNullable(channels.get(key)) - .map(v -> v.channel) - .orElseGet(() -> { - var channel = supplier.apply(key); - channels.put(key, new Both<>(channel, null)); - return channel; - })); - } + public C computeIfAbsent(K key, Function supplier) { + return entries.computeIfAbsent(key, + k -> new Value<>(supplier.apply(k), null)).channel(); } /** @@ -183,121 +148,17 @@ public class ChannelManager { * @return the channel manager */ public ChannelManager associate(K key, A data) { - synchronized (channels) { - Optional.ofNullable(channels.get(key)) - .ifPresent(v -> v.associated = data); - } + Optional.ofNullable(entries.computeIfPresent(key, + (k, existing) -> new Value<>(existing.channel(), data))); return this; } - /** - * Return the data associated with the entry for the channel. - * - * @param key the key - * @return the data - */ - public Optional associated(K key) { - return both(key).map(b -> b.associated); - } - - /** - * Returns all associated data. - * - * @return the collection - */ - public Collection associated() { - synchronized (channels) { - return channels.values().stream() - .filter(v -> v.associated != null) - .map(v -> v.associated).toList(); - } - } - /** * Removes the channel with the given name. * * @param name the name */ public void remove(String name) { - synchronized (channels) { - channels.remove(name); - } - } - - /** - * Returns all known keys. - * - * @return the sets the - */ - public Set keys() { - return channels.keySet(); - } - - /** - * Returns a read only view of this channel manager. The methods - * that usually create a new entry refrain from doing so. The - * methods that change the value of channel and {@link #remove(String)} - * do nothing. The associated data, however, can still be changed. - * - * @return the channel manager - */ - public ChannelManager fixed() { - if (readOnly == null) { - readOnly = new ChannelManager<>(supplier) { - - @Override - public Optional> both(K key) { - return ChannelManager.this.both(key); - } - - @Override - public ChannelManager put(K key, C channel, - A associated) { - return associate(key, associated); - } - - @Override - public Optional getChannel(K key) { - return ChannelManager.this.channel(key); - } - - @Override - public Optional getChannel(K key, Function supplier) { - return ChannelManager.this.channel(key); - } - - @Override - public ChannelManager associate(K key, A data) { - return ChannelManager.this.associate(key, data); - } - - @Override - public Optional associated(K key) { - return ChannelManager.this.associated(key); - } - - @Override - public Collection associated() { - return ChannelManager.this.associated(); - } - - @Override - public void remove(String name) { - // Do nothing - } - - @Override - public Set keys() { - return ChannelManager.this.keys(); - } - - @Override - public ChannelManager fixed() { - return ChannelManager.this.fixed(); - } - - }; - } - return readOnly; + entries.remove(name); } } diff --git a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelCache.java b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelTracker.java similarity index 52% rename from org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelCache.java rename to org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelTracker.java index 1e6d031..8a41908 100644 --- a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelCache.java +++ b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ChannelTracker.java @@ -19,6 +19,7 @@ package org.jdrupes.vmoperator.manager.events; import java.lang.ref.WeakReference; +import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.Optional; @@ -27,20 +28,30 @@ import java.util.concurrent.ConcurrentHashMap; import org.jgrapes.core.Channel; /** - * A channel manager that tracks mappings from a key to a channel using - * "add/remove" (or "open/close") events and the channels on which they - * are delivered. + * Used to track mapping from a key to a channel. Entries must + * be maintained by handlers for "add/remove" (or "open/close") + * events delivered on the channels that are to be + * made available by the tracker. + * + * The channels are stored in the dictionary using {@link WeakReference}s. + * Removing entries is therefore best practice but not an absolute necessity + * as entries for cleared references are removed when one of the methods + * {@link #values()}, {@link #channels()} or {@link #associated()} is called. * * @param the key type * @param the channel type * @param the type of the associated data */ -public class ChannelCache { +public class ChannelTracker + implements ChannelDictionary { - private final Map> channels = new ConcurrentHashMap<>(); + private final Map> entries = new ConcurrentHashMap<>(); /** - * Helper + * Combines the channel and associated data. + * + * @param the generic type + * @param the generic type */ @SuppressWarnings("PMD.ShortClassName") private static class Data { @@ -57,32 +68,24 @@ public class ChannelCache { } } - /** - * Combines the channel and the associated data. - * - * @param the generic type - * @param the generic type - */ - @SuppressWarnings("PMD.ShortClassName") - public static class Both { + @Override + public Set keys() { + return entries.keySet(); + } - /** The channel. */ - public C channel; - - /** The associated. */ - public A associated; - - /** - * Instantiates a new both. - * - * @param channel the channel - * @param associated the associated - */ - public Both(C channel, A associated) { - super(); - this.channel = channel; - this.associated = associated; + @Override + public Collection> values() { + var result = new ArrayList>(); + for (var itr = entries.entrySet().iterator(); itr.hasNext();) { + var value = itr.next().getValue(); + var channel = value.channel.get(); + if (channel == null) { + itr.remove(); + continue; + } + result.add(new Value<>(channel, value.associated)); } + return result; } /** @@ -92,20 +95,18 @@ public class ChannelCache { * @param key the key * @return the result */ - public Optional> both(K key) { - synchronized (channels) { - var value = channels.get(key); - if (value == null) { - return Optional.empty(); - } - var channel = value.channel.get(); - if (channel == null) { - // Cleanup old reference - channels.remove(key); - return Optional.empty(); - } - return Optional.of(new Both<>(channel, value.associated)); + public Optional> value(K key) { + var value = entries.get(key); + if (value == null) { + return Optional.empty(); } + var channel = value.channel.get(); + if (channel == null) { + // Cleanup old reference + entries.remove(key); + return Optional.empty(); + } + return Optional.of(new Value<>(channel, value.associated)); } /** @@ -116,10 +117,10 @@ public class ChannelCache { * @param associated the associated * @return the channel manager */ - public ChannelCache put(K key, C channel, A associated) { + public ChannelTracker put(K key, C channel, A associated) { Data data = new Data<>(channel); data.associated = associated; - channels.put(key, data); + entries.put(key, data); return this; } @@ -130,22 +131,11 @@ public class ChannelCache { * @param channel the channel * @return the channel manager */ - public ChannelCache put(K key, C channel) { + public ChannelTracker put(K key, C channel) { put(key, channel, null); return this; } - /** - * Returns the channel registered for the key or an empty optional - * if no mapping exists. - * - * @param key the key - * @return the optional - */ - public Optional channel(K key) { - return both(key).map(b -> b.channel); - } - /** * Associate the entry for the channel with the given data. The entry * for the channel must already exist. @@ -154,54 +144,18 @@ public class ChannelCache { * @param data the data * @return the channel manager */ - public ChannelCache associate(K key, A data) { - synchronized (channels) { - Optional.ofNullable(channels.get(key)) - .ifPresent(v -> v.associated = data); - } + public ChannelTracker associate(K key, A data) { + Optional.ofNullable(entries.get(key)) + .ifPresent(v -> v.associated = data); return this; } - /** - * Return the data associated with the entry for the channel. - * - * @param key the key - * @return the data - */ - public Optional associated(K key) { - return both(key).map(b -> b.associated); - } - - /** - * Returns all associated data. - * - * @return the collection - */ - public Collection associated() { - synchronized (channels) { - return channels.values().stream() - .filter(v -> v.channel.get() != null && v.associated != null) - .map(v -> v.associated).toList(); - } - } - /** * Removes the channel with the given name. * * @param name the name */ public void remove(String name) { - synchronized (channels) { - channels.remove(name); - } - } - - /** - * Returns all known keys. - * - * @return the sets the - */ - public Set keys() { - return channels.keySet(); + entries.remove(name); } } diff --git a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ServiceChanged.java b/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ServiceChanged.java deleted file mode 100644 index a8008e0..0000000 --- a/org.jdrupes.vmoperator.manager.events/src/org/jdrupes/vmoperator/manager/events/ServiceChanged.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * VM-Operator - * Copyright (C) 2024 Michael N. Lipp - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package org.jdrupes.vmoperator.manager.events; - -import io.kubernetes.client.openapi.models.V1Service; -import org.jdrupes.vmoperator.common.K8sObserver.ResponseType; -import org.jgrapes.core.Channel; -import org.jgrapes.core.Components; -import org.jgrapes.core.Event; - -/** - * Indicates that a service has changed. - */ -@SuppressWarnings("PMD.DataClass") -public class ServiceChanged extends Event { - - private final ResponseType type; - private final V1Service service; - - /** - * Initializes a new service changed event. - * - * @param type the type - * @param service the service - */ - public ServiceChanged(ResponseType type, V1Service service) { - this.type = type; - this.service = service; - } - - /** - * Returns the type. - * - * @return the type - */ - public ResponseType type() { - return type; - } - - /** - * Gets the service. - * - * @return the service - */ - public V1Service service() { - return service; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append(Components.objectName(this)).append(" [") - .append(service.getMetadata().getName()).append(' ').append(type); - if (channels() != null) { - builder.append(", channels=").append(Channel.toString(channels())); - } - builder.append(']'); - return builder.toString(); - } -} diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java index 0c9b08c..43f4287 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/AbstractMonitor.java @@ -27,14 +27,11 @@ import io.kubernetes.client.util.generic.options.ListOptions; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import org.jdrupes.vmoperator.common.K8s; import org.jdrupes.vmoperator.common.K8sClient; import org.jdrupes.vmoperator.common.K8sObserver; -import org.jdrupes.vmoperator.common.K8sObserver.ResponseType; -import org.jdrupes.vmoperator.manager.events.ChannelManager; import org.jdrupes.vmoperator.manager.events.Exit; import org.jgrapes.core.Channel; import org.jgrapes.core.Component; @@ -45,7 +42,11 @@ import org.jgrapes.core.events.Stop; import org.jgrapes.util.events.ConfigurationUpdate; /** - * A base class for monitoring VM related resources. + * A base class for monitoring VM related resources. When started, + * it creates observers for all versions of the the {@link APIResource} + * configured by {@link #context(APIResource)}. The APIResource is not + * passed to the constructor because in some cases it has to be + * evaluated lazily. * * @param the object type for the context * @param the object list type for the context @@ -61,15 +62,17 @@ public abstract class AbstractMonitor channelManager; /** * Initializes the instance. * * @param componentChannel the component channel + * @param objectClass the class of the Kubernetes object to watch + * @param objectListClass the class of the list of Kubernetes objects + * to watch */ - protected AbstractMonitor(Channel componentChannel, Class objectClass, - Class objectListClass) { + protected AbstractMonitor(Channel componentChannel, + Class objectClass, Class objectListClass) { super(componentChannel); this.objectClass = objectClass; this.objectListClass = objectListClass; @@ -155,27 +158,6 @@ public abstract class AbstractMonitor channelManager() { - return channelManager; - } - - /** - * Sets the channel manager. - * - * @param channelManager the channel manager - * @return the abstract monitor - */ - public AbstractMonitor - channelManager(ChannelManager channelManager) { - this.channelManager = channelManager; - return this; - } - /** * Looks for a key "namespace" in the configuration and, if found, * sets the namespace to its value. @@ -193,7 +175,7 @@ public abstract class AbstractMonitor { handleChange(c, r); - if (ResponseType.valueOf(r.type) == ResponseType.DELETED) { - channelManager.remove(r.object.getMetadata().getName()); - } }).onTerminated((o, t) -> { if (observerCounter.decrementAndGet() == 0) { unregisterAsGenerator(); @@ -255,7 +234,8 @@ public abstract class AbstractMonitor change); - - /** - * Returns the {@link Channel} for the given name. - * - * @param name the name - * @return the channel used for events related to the specified object - */ - protected Optional channel(String name) { - return channelManager.getChannel(name); - } } diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java index 86e3751..effc938 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java @@ -100,9 +100,8 @@ public class Controller extends Component { return null; } }); - attach(new VmMonitor(channel()).channelManager(chanMgr)); - attach(new DisplaySecretMonitor(channel()) - .channelManager(chanMgr.fixed())); + attach(new VmMonitor(channel(), chanMgr)); + attach(new DisplaySecretMonitor(channel(), chanMgr)); // Currently, we don't use the IP assigned by the load balancer // to access the VM's console. Might change in the future. // attach(new ServiceMonitor(channel()).channelManager(chanMgr)); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretMonitor.java index c113253..fa0bbf0 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretMonitor.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretMonitor.java @@ -44,6 +44,7 @@ import org.jdrupes.vmoperator.common.K8sV1SecretStub; import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET; import static org.jdrupes.vmoperator.manager.Constants.DATA_DISPLAY_PASSWORD; import static org.jdrupes.vmoperator.manager.Constants.DATA_PASSWORD_EXPIRY; +import org.jdrupes.vmoperator.manager.events.ChannelDictionary; import org.jdrupes.vmoperator.manager.events.GetDisplayPassword; import org.jdrupes.vmoperator.manager.events.VmChannel; import org.jdrupes.vmoperator.manager.events.VmDefChanged; @@ -68,14 +69,18 @@ public class DisplaySecretMonitor private int passwordValidity = 10; private final List pendingGets = Collections.synchronizedList(new LinkedList<>()); + private final ChannelDictionary channelDictionary; /** * Instantiates a new display secrets monitor. * * @param componentChannel the component channel + * @param channelDictionary the channel dictionary */ - public DisplaySecretMonitor(Channel componentChannel) { + public DisplaySecretMonitor(Channel componentChannel, + ChannelDictionary channelDictionary) { super(componentChannel, V1Secret.class, V1SecretList.class); + this.channelDictionary = channelDictionary; context(K8sV1SecretStub.CONTEXT); ListOptions options = new ListOptions(); options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + "," @@ -116,7 +121,7 @@ public class DisplaySecretMonitor if (vmName == null) { return; } - var channel = channel(vmName).orElse(null); + var channel = channelDictionary.channel(vmName).orElse(null); if (channel == null || channel.vmDefinition() == null) { return; } @@ -248,6 +253,7 @@ public class DisplaySecretMonitor * @param channel the channel */ @Handler + @SuppressWarnings("PMD.AvoidSynchronizedStatement") public void onVmDefChanged(VmDefChanged event, Channel channel) { synchronized (pendingGets) { String vmName = event.vmDefinition().metadata().getName(); diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ServiceMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ServiceMonitor.java deleted file mode 100644 index bd5635e..0000000 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ServiceMonitor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * VM-Operator - * Copyright (C) 2024 Michael N. Lipp - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package org.jdrupes.vmoperator.manager; - -import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.models.V1Service; -import io.kubernetes.client.openapi.models.V1ServiceList; -import io.kubernetes.client.util.Watch.Response; -import io.kubernetes.client.util.generic.options.ListOptions; -import java.io.IOException; -import static org.jdrupes.vmoperator.common.Constants.APP_NAME; -import org.jdrupes.vmoperator.common.K8sClient; -import org.jdrupes.vmoperator.common.K8sObserver.ResponseType; -import org.jdrupes.vmoperator.common.K8sV1ServiceStub; -import org.jdrupes.vmoperator.manager.events.ServiceChanged; -import org.jdrupes.vmoperator.manager.events.VmChannel; -import org.jgrapes.core.Channel; - -/** - * Watches for changes of services. - */ -@SuppressWarnings("PMD.DataflowAnomalyAnalysis") -public class ServiceMonitor - extends AbstractMonitor { - - /** - * Instantiates a new display secrets monitor. - * - * @param componentChannel the component channel - */ - public ServiceMonitor(Channel componentChannel) { - super(componentChannel, V1Service.class, V1ServiceList.class); - context(K8sV1ServiceStub.CONTEXT); - ListOptions options = new ListOptions(); - options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME); - options(options); - } - - @Override - protected void prepareMonitoring() throws IOException, ApiException { - client(new K8sClient()); - } - - @Override - protected void handleChange(K8sClient client, Response change) { - String vmName = change.object.getMetadata().getLabels() - .get("app.kubernetes.io/instance"); - if (vmName == null) { - return; - } - var channel = channel(vmName).orElse(null); - if (channel == null || channel.vmDefinition() == null) { - return; - } - channel.pipeline().fire(new ServiceChanged( - ResponseType.valueOf(change.type), change.object), channel); - } -} diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java index e049b17..0ad5017 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java @@ -43,10 +43,12 @@ import org.jdrupes.vmoperator.common.VmDefinitionStub; import static org.jdrupes.vmoperator.manager.Constants.APP_NAME; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_KIND_VM; import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME; +import org.jdrupes.vmoperator.manager.events.ChannelManager; import org.jdrupes.vmoperator.manager.events.VmChannel; import org.jdrupes.vmoperator.manager.events.VmDefChanged; import org.jdrupes.vmoperator.util.GsonPtr; import org.jgrapes.core.Channel; +import org.jgrapes.core.Event; /** * Watches for changes of VM definitions. @@ -55,14 +57,19 @@ import org.jgrapes.core.Channel; public class VmMonitor extends AbstractMonitor { + private final ChannelManager channelManager; + /** * Instantiates a new VM definition watcher. * * @param componentChannel the component channel + * @param channelManager the channel manager */ - public VmMonitor(Channel componentChannel) { + public VmMonitor(Channel componentChannel, + ChannelManager channelManager) { super(componentChannel, VmDefinitionModel.class, VmDefinitionModels.class); + this.channelManager = channelManager; } @Override @@ -107,10 +114,7 @@ public class VmMonitor extends protected void handleChange(K8sClient client, Watch.Response response) { V1ObjectMeta metadata = response.object.getMetadata(); - VmChannel channel = channel(metadata.getName()).orElse(null); - if (channel == null) { - return; - } + VmChannel channel = channelManager.channelGet(metadata.getName()); // Get full definition and associate with channel as backup var vmDef = response.object; @@ -132,13 +136,24 @@ public class VmMonitor extends () -> "Cannot get model for " + response.object.getMetadata()); return; } + if (ResponseType.valueOf(response.type) == ResponseType.DELETED) { + channelManager.remove(metadata.getName()); + } - // Create and fire event + // Create and fire changed event. Remove channel from channel + // manager on completion. channel.pipeline() - .fire(new VmDefChanged(ResponseType.valueOf(response.type), - channel.setGeneration( - response.object.getMetadata().getGeneration()), - vmDef), channel); + .fire(Event.onCompletion( + new VmDefChanged(ResponseType.valueOf(response.type), + channel.setGeneration( + response.object.getMetadata().getGeneration()), + vmDef), + e -> { + if (e.type() == ResponseType.DELETED) { + channelManager + .remove(e.vmDefinition().metadata().getName()); + } + }), channel); } private VmDefinitionModel getModel(K8sClient client, diff --git a/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/TimeSeries.java b/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/TimeSeries.java index ee1667c..62bdf55 100644 --- a/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/TimeSeries.java +++ b/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/TimeSeries.java @@ -51,7 +51,8 @@ public class TimeSeries { * @param numbers the numbers * @return the time series */ - @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") + @SuppressWarnings({ "PMD.AvoidLiteralsInIfCondition", + "PMD.AvoidSynchronizedStatement" }) public TimeSeries add(Instant time, Number... numbers) { var newEntry = new Entry(time, numbers); boolean nothingNew = false; @@ -83,6 +84,7 @@ public class TimeSeries { * * @return the list */ + @SuppressWarnings("PMD.AvoidSynchronizedStatement") public List entries() { synchronized (data) { return new ArrayList<>(data); diff --git a/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/VmConlet.java b/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/VmConlet.java index a8bb1ae..b8ff79f 100644 --- a/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/VmConlet.java +++ b/org.jdrupes.vmoperator.vmconlet/src/org/jdrupes/vmoperator/vmconlet/VmConlet.java @@ -30,14 +30,14 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.time.Duration; import java.time.Instant; -import java.util.HashSet; +import java.util.EnumSet; import java.util.Optional; import java.util.Set; import org.jdrupes.json.JsonBeanDecoder; import org.jdrupes.json.JsonDecodeException; import org.jdrupes.vmoperator.common.K8sObserver; import org.jdrupes.vmoperator.common.VmDefinitionModel; -import org.jdrupes.vmoperator.manager.events.ChannelCache; +import org.jdrupes.vmoperator.manager.events.ChannelTracker; import org.jdrupes.vmoperator.manager.events.ModifyVm; import org.jdrupes.vmoperator.manager.events.VmChannel; import org.jdrupes.vmoperator.manager.events.VmDefChanged; @@ -68,8 +68,8 @@ public class VmConlet extends FreeMarkerConlet { private static final Set MODES = RenderMode.asSet( RenderMode.Preview, RenderMode.View); - private final ChannelCache channelManager = new ChannelCache<>(); + private final ChannelTracker channelTracker = new ChannelTracker<>(); private final TimeSeries summarySeries = new TimeSeries(Duration.ofDays(1)); private Summary cachedSummary; @@ -128,7 +128,7 @@ public class VmConlet extends FreeMarkerConlet { protected Set doRenderConlet(RenderConletRequestBase event, ConsoleConnection channel, String conletId, VmsModel conletState) throws Exception { - Set renderedAs = new HashSet<>(); + Set renderedAs = EnumSet.noneOf(RenderMode.class); boolean sendVmInfos = false; if (event.renderAs().contains(RenderMode.Preview)) { Template tpl @@ -160,7 +160,7 @@ public class VmConlet extends FreeMarkerConlet { sendVmInfos = true; } if (sendVmInfos) { - for (var vmDef : channelManager.associated()) { + for (var vmDef : channelTracker.associated()) { var def = JsonBeanDecoder.create(vmDef.data().toString()) .readObject(); @@ -188,7 +188,7 @@ public class VmConlet extends FreeMarkerConlet { throws JsonDecodeException, IOException { var vmName = event.vmDefinition().getMetadata().getName(); if (event.type() == K8sObserver.ResponseType.DELETED) { - channelManager.remove(vmName); + channelTracker.remove(vmName); for (var entry : conletIdsByConsoleConnection().entrySet()) { for (String conletId : entry.getValue()) { entry.getKey().respond(new NotifyConletView(type(), @@ -198,7 +198,7 @@ public class VmConlet extends FreeMarkerConlet { } else { var vmDef = new VmDefinitionModel(channel.client().getJSON() .getGson(), cleanup(event.vmDefinition().data())); - channelManager.put(vmName, channel, vmDef); + channelTracker.put(vmName, channel, vmDef); var def = JsonBeanDecoder.create(vmDef.data().toString()) .readObject(); for (var entry : conletIdsByConsoleConnection().entrySet()) { @@ -321,7 +321,7 @@ public class VmConlet extends FreeMarkerConlet { return cachedSummary; } Summary summary = new Summary(); - for (var vmDef : channelManager.associated()) { + for (var vmDef : channelTracker.associated()) { summary.totalVms += 1; var status = GsonPtr.to(vmDef.data()).to("status"); summary.usedCpus += status.getAsInt("cpus").orElse(0); @@ -347,7 +347,7 @@ public class VmConlet extends FreeMarkerConlet { throws Exception { event.stop(); var vmName = event.params().asString(0); - var vmChannel = channelManager.channel(vmName).orElse(null); + var vmChannel = channelTracker.channel(vmName).orElse(null); if (vmChannel == null) { return; } diff --git a/org.jdrupes.vmoperator.vmviewer/src/org/jdrupes/vmoperator/vmviewer/VmViewer.java b/org.jdrupes.vmoperator.vmviewer/src/org/jdrupes/vmoperator/vmviewer/VmViewer.java index e1e5c6d..db89b81 100644 --- a/org.jdrupes.vmoperator.vmviewer/src/org/jdrupes/vmoperator/vmviewer/VmViewer.java +++ b/org.jdrupes.vmoperator.vmviewer/src/org/jdrupes/vmoperator/vmviewer/VmViewer.java @@ -53,7 +53,7 @@ import org.jdrupes.vmoperator.common.K8sDynamicModel; import org.jdrupes.vmoperator.common.K8sObserver; import org.jdrupes.vmoperator.common.VmDefinitionModel; import org.jdrupes.vmoperator.common.VmDefinitionModel.Permission; -import org.jdrupes.vmoperator.manager.events.ChannelCache; +import org.jdrupes.vmoperator.manager.events.ChannelTracker; import org.jdrupes.vmoperator.manager.events.GetDisplayPassword; import org.jdrupes.vmoperator.manager.events.ModifyVm; import org.jdrupes.vmoperator.manager.events.ResetVm; @@ -122,8 +122,8 @@ public class VmViewer extends FreeMarkerConlet { RenderMode.Preview, RenderMode.Edit); private static final Set MODES_FOR_GENERATED = RenderMode.asSet( RenderMode.Preview, RenderMode.StickyPreview); - private final ChannelCache channelManager = new ChannelCache<>(); + private final ChannelTracker channelTracker = new ChannelTracker<>(); private static ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); private Class preferredIpVersion = Inet4Address.class; @@ -349,7 +349,7 @@ public class VmViewer extends FreeMarkerConlet { // Remove conlet if definition has been removed if (model.vmName() != null - && !channelManager.associated(model.vmName()).isPresent()) { + && !channelTracker.associated(model.vmName()).isPresent()) { channel.respond( new DeleteConlet(conletId, Collections.emptySet())); return Collections.emptySet(); @@ -357,7 +357,7 @@ public class VmViewer extends FreeMarkerConlet { // Don't render if user has not at least one permission if (model.vmName() != null - && channelManager.associated(model.vmName()) + && channelTracker.associated(model.vmName()) .map(d -> permissions(d, channel.session()).isEmpty()) .orElse(true)) { return Collections.emptySet(); @@ -395,7 +395,7 @@ public class VmViewer extends FreeMarkerConlet { } private List accessibleVms(ConsoleConnection channel) { - return channelManager.associated().stream() + return channelTracker.associated().stream() .filter(d -> !permissions(d, channel.session()).isEmpty()) .map(d -> d.getMetadata().getName()).sorted().toList(); } @@ -419,7 +419,7 @@ public class VmViewer extends FreeMarkerConlet { if (Strings.isNullOrEmpty(model.vmName())) { return; } - channelManager.associated(model.vmName()).ifPresent(vmDef -> { + channelTracker.associated(model.vmName()).ifPresent(vmDef -> { try { var def = JsonBeanDecoder.create(vmDef.data().toString()) .readObject(); @@ -465,9 +465,9 @@ public class VmViewer extends FreeMarkerConlet { .remove("managedFields"); var vmName = vmDef.getMetadata().getName(); if (event.type() == K8sObserver.ResponseType.DELETED) { - channelManager.remove(vmName); + channelTracker.remove(vmName); } else { - channelManager.put(vmName, channel, vmDef); + channelTracker.put(vmName, channel, vmDef); } for (var entry : conletIdsByConsoleConnection().entrySet()) { var connection = entry.getKey(); @@ -502,12 +502,12 @@ public class VmViewer extends FreeMarkerConlet { // Handle command for selected VM var both = Optional.ofNullable(model.vmName()) - .flatMap(vm -> channelManager.both(vm)); + .flatMap(vm -> channelTracker.value(vm)); if (both.isEmpty()) { return; } - var vmChannel = both.get().channel; - var vmDef = both.get().associated; + var vmChannel = both.get().channel(); + var vmDef = both.get().associated(); var vmName = vmDef.metadata().getName(); var perms = permissions(vmDef, channel.session()); var resourceBundle = resourceBundle(channel.locale()); @@ -556,7 +556,7 @@ public class VmViewer extends FreeMarkerConlet { private void openConsole(String vmName, ConsoleConnection connection, ViewerModel model, String password) { - var vmDef = channelManager.associated(vmName).orElse(null); + var vmDef = channelTracker.associated(vmName).orElse(null); if (vmDef == null) { return; } From 452e0604cacf5765129a780b5f97079eb015af98 Mon Sep 17 00:00:00 2001 From: "Michael N. Lipp" Date: Sun, 6 Oct 2024 14:07:09 +0200 Subject: [PATCH 3/3] Update version. --- deploy/vmop-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/vmop-deployment.yaml b/deploy/vmop-deployment.yaml index 648cc39..33d9674 100644 --- a/deploy/vmop-deployment.yaml +++ b/deploy/vmop-deployment.yaml @@ -20,7 +20,7 @@ spec: containers: - name: vm-operator image: >- - ghcr.io/mnlipp/org.jdrupes.vmoperator.manager:latest + ghcr.io/mnlipp/org.jdrupes.vmoperator.manager:3.4.0 volumeMounts: - name: config mountPath: /etc/opt/vmoperator