mnl/VM-Operator
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 4 Wiki Activity Actions

Use less gson internally.

Browse source
This commit is contained in:
Michael Lipp 2024-11-08 16:48:07 +00:00
parent 8e692a03fe
commit c8781c2d8e
33 changed files with 1405 additions and 905 deletions
Download patch file Download diff file Expand all files Collapse all files

139
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml
View file

@ -1,141 +1,142 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
namespace: ${ cr.namespace() }
name: ${ cr.name() }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
vmoperator.jdrupes.org/version: ${ managerVersion }
ownerReferences:
- apiVersion: ${ cr.apiVersion.asString }
- apiVersion: ${ cr.apiVersion }
kind: ${ constants.VM_OP_KIND_VM }
name: ${ cr.metadata.name.asString }
uid: ${ cr.metadata.uid.asString }
name: ${ cr.name() }
uid: ${ cr.metadata().getUid() }
controller: false
data:
config.yaml: |
"/Runner":
# The directory used to store data files. Defaults to (depending on
# values available):
# * $XDG_DATA_HOME/vmrunner/${ cr.metadata.name.asString }
# * $HOME/.local/share/vmrunner/${ cr.metadata.name.asString }
# * ./${ cr.metadata.name.asString }
# * $XDG_DATA_HOME/vmrunner/${ cr.name() }
# * $HOME/.local/share/vmrunner/${ cr.name() }
# * ./${ cr.name() }
dataDir: /var/local/vm-data
# The directory used to store runtime files. Defaults to (depending on
# values available):
# * $XDG_RUNTIME_DIR/vmrunner/${ cr.metadata.name.asString }
# * /tmp/$USER/vmrunner/${ cr.metadata.name.asString }
# * /tmp/vmrunner/${ cr.metadata.name.asString }
# runtimeDir: "$XDG_RUNTIME_DIR/vmrunner/${ cr.metadata.name.asString }"
# * $XDG_RUNTIME_DIR/vmrunner/${ cr.name() }
# * /tmp/$USER/vmrunner/${ cr.name() }
# * /tmp/vmrunner/${ cr.name() }
# runtimeDir: "$XDG_RUNTIME_DIR/vmrunner/${ cr.name() }"
<#assign spec = cr.spec() />
# The template to use. Resolved relative to /usr/share/vmrunner/templates.
# template: "Standard-VM-latest.ftl.yaml"
<#if cr.spec.runnerTemplate?? && cr.spec.runnerTemplate.source?? >
template: ${ cr.spec.runnerTemplate.source.asString }
<#if spec.runnerTemplate?? && spec.runnerTemplate.source?? >
template: ${ cm.spec().runnerTemplate.source }
</#if>
# The template is copied to the data diretory when the VM starts for
# the first time. Subsequent starts use the copy unless this option is set.
<#if cr.spec.runnerTemplate?? && cr.spec.runnerTemplate.update?? >
updateTemplate: ${ cr.spec.runnerTemplate.update.asBoolean?c }
<#if spec.runnerTemplate?? && spec.runnerTemplate.update?? >
updateTemplate: ${ spec.runnerTemplate.update?c }
</#if>
# Whether a shutdown initiated by the guest stops the pod deployment
guestShutdownStops: ${ cr.spec.guestShutdownStops!false?c }
guestShutdownStops: ${ (spec.guestShutdownStops!false)?c }
# When incremented, the VM is reset. The value has no default value,
# i.e. if you start the VM without a value for this property, and
# decide to trigger a reset later, you have to first set the value
# and then inrement it.
resetCounter: ${ cr.resetCount }
resetCounter: ${ cr.extra("resetCount")?c }
# Forward the cloud-init data if provided
<#if cr.spec.cloudInit??>
<#if spec.cloudInit??>
cloudInit:
<#if cr.spec.cloudInit.metaData??>
metaData: ${ cr.spec.cloudInit.metaData.toString() }
<#if spec.cloudInit.metaData??>
metaData: ${ toJson(adjustCloudInitMeta(spec.cloudInit.metaData, cr.metadata())) }
<#else>
metaData: {}
</#if>
<#if cr.spec.cloudInit.userData??>
userData: ${ cr.spec.cloudInit.userData.toString() }
<#if spec.cloudInit.userData??>
userData: ${ toJson(spec.cloudInit.userData) }
<#else>
userData: {}
</#if>
<#if cr.spec.cloudInit.networkConfig??>
networkConfig: ${ cr.spec.cloudInit.networkConfig.toString() }
<#if spec.cloudInit.networkConfig??>
networkConfig: ${ toJson(spec.cloudInit.networkConfig) }
</#if>
</#if>
# Define the VM (required)
vm:
# The VM's name (required)
name: ${ cr.metadata.name.asString }
name: ${ cr.name() }
# The machine's uuid. If none is specified, a uuid is generated
# and stored in the data directory. If the uuid is important
# (e.g. because licenses depend on it) it is recommaned to specify
# it here explicitly or to carefully backup the data directory.
# uuid: "generated uuid"
<#if cr.spec.vm.machineUuid??>
uuid: "${ cr.spec.vm.machineUuid.asString }"
<#if spec.vm.machineUuid??>
uuid: "${ spec.vm.machineUuid }"
</#if>
# Whether to provide a software TPM (defaults to false)
# useTpm: false
useTpm: ${ cr.spec.vm.useTpm.asBoolean?c }
useTpm: ${ spec.vm.useTpm?c }
# How to boot (see https://github.com/mnlipp/VM-Operator/blob/main/org.jdrupes.vmoperator.runner.qemu/resources/org/jdrupes/vmoperator/runner/qemu/defaults.yaml):
# * bios
# * uefi[-4m]
# * secure[-4m]
firmware: ${ cr.spec.vm.firmware.asString }
firmware: ${ spec.vm.firmware }
# Whether to show a boot menu.
# bootMenu: false
bootMenu: ${ cr.spec.vm.bootMenu.asBoolean?c }
bootMenu: ${ spec.vm.bootMenu?c }
# When terminating, a graceful powerdown is attempted. If it
# doesn't succeed within the given timeout (seconds) SIGTERM
# is sent to Qemu.
# powerdownTimeout: 900
powerdownTimeout: ${ cr.spec.vm.powerdownTimeout.asLong?c }
powerdownTimeout: ${ spec.vm.powerdownTimeout?c }
# CPU settings
cpuModel: ${ cr.spec.vm.cpuModel.asString }
cpuModel: ${ spec.vm.cpuModel }
# Setting maximumCpus to 1 omits the "-smp" options. The defaults (0)
# cause the corresponding property to be omitted from the "-smp" option.
# If currentCpus is greater than maximumCpus, the latter is adjusted.
<#if cr.spec.vm.maximumCpus?? >
maximumCpus: ${ parseQuantity(cr.spec.vm.maximumCpus.asString)?c }
<#if spec.vm.maximumCpus?? >
maximumCpus: ${ parseQuantity(spec.vm.maximumCpus)?c }
</#if>
<#if cr.spec.vm.cpuTopology?? >
sockets: ${ cr.spec.vm.cpuTopology.sockets.asInt?c }
diesPerSocket: ${ cr.spec.vm.cpuTopology.diesPerSocket.asInt?c }
coresPerDie: ${ cr.spec.vm.cpuTopology.coresPerDie.asInt?c }
threadsPerCore: ${ cr.spec.vm.cpuTopology.threadsPerCore.asInt?c }
<#if spec.vm.cpuTopology?? >
sockets: ${ spec.vm.cpuTopology.sockets?c }
diesPerSocket: ${ spec.vm.cpuTopology.diesPerSocket?c }
coresPerDie: ${ spec.vm.cpuTopology.coresPerDie?c }
threadsPerCore: ${ spec.vm.cpuTopology.threadsPerCore?c }
</#if>
<#if cr.spec.vm.currentCpus?? >
currentCpus: ${ parseQuantity(cr.spec.vm.currentCpus.asString)?c }
<#if spec.vm.currentCpus?? >
currentCpus: ${ parseQuantity(spec.vm.currentCpus)?c }
</#if>
# RAM settings
# Maximum defaults to 1G
maximumRam: "${ formatMemory(parseQuantity(cr.spec.vm.maximumRam.asString)) }"
<#if cr.spec.vm.currentRam?? >
currentRam: "${ formatMemory(parseQuantity(cr.spec.vm.currentRam.asString)) }"
maximumRam: "${ formatMemory(parseQuantity(spec.vm.maximumRam)) }"
<#if spec.vm.currentRam?? >
currentRam: "${ formatMemory(parseQuantity(spec.vm.currentRam)) }"
</#if>
# RTC settings.
# rtcBase: utc
# rtcClock: rt
rtcBase: ${ cr.spec.vm.rtcBase.asString }
rtcClock: ${ cr.spec.vm.rtcClock.asString }
rtcBase: ${ spec.vm.rtcBase }
rtcClock: ${ spec.vm.rtcClock }
# Network settings
# Supported types are "tap" and "user" (for debugging). Type "user"
@ -147,19 +148,19 @@ data:
# mac: (undefined)
network:
<#assign nwCounter = 0/>
<#list cr.spec.vm.networks.asList() as itf>
<#list spec.vm.networks as itf>
<#if itf.tap??>
- type: tap
device: ${ itf.tap.device.asString }
bridge: ${ itf.tap.bridge.asString }
device: ${ itf.tap.device }
bridge: ${ itf.tap.bridge }
<#if itf.tap.mac??>
mac: "${ itf.tap.mac.asString }"
mac: "${ itf.tap.mac }"
</#if>
<#elseif itf.user??>
- type: user
device: ${ itf.user.device.asString }
device: ${ itf.user.device }
<#if itf.user.net??>
net: "${ itf.user.net.asString }"
net: "${ itf.user.net }"
</#if>
</#if>
<#assign nwCounter += 1/>
@ -175,11 +176,11 @@ data:
# file: (undefined)
drives:
<#assign drvCounter = 0/>
<#list cr.spec.vm.disks.asList() as disk>
<#list spec.vm.disks as disk>
<#if disk.volumeClaimTemplate??
&& disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.name??>
<#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
<#assign diskName = disk.volumeClaimTemplate.metadata.name + "-disk">
<#else>
<#assign diskName = "disk-" + drvCounter>
</#if>
@ -187,33 +188,33 @@ data:
- type: raw
resource: /dev/${ diskName }
<#if disk.bootindex??>
bootindex: ${ disk.bootindex.asInt?c }
bootindex: ${ disk.bootindex?c }
</#if>
<#assign drvCounter = drvCounter + 1/>
</#if>
<#if disk.cdrom??>
- type: ide-cd
file: "${ disk.cdrom.image.asString }"
file: "${ imageLocation(disk.cdrom.image) }"
<#if disk.bootindex??>
bootindex: ${ disk.bootindex.asInt?c }
bootindex: ${ disk.bootindex?c }
</#if>
</#if>
</#list>
display:
<#if cr.spec.vm.display.outputs?? >
outputs: ${ cr.spec.vm.display.outputs.asInt?c }
<#if spec.vm.display.outputs?? >
outputs: ${ spec.vm.display.outputs?c }
</#if>
<#if cr.spec.vm.display.spice??>
<#if spec.vm.display.spice??>
spice:
port: ${ cr.spec.vm.display.spice.port.asInt?c }
<#if cr.spec.vm.display.spice.ticket??>
ticket: "${ cr.spec.vm.display.spice.ticket.asString }"
port: ${ spec.vm.display.spice.port?c }
<#if spec.vm.display.spice.ticket??>
ticket: "${ spec.vm.display.spice.ticket }"
</#if>
<#if cr.spec.vm.display.spice.streamingVideo??>
streaming-video: "${ cr.spec.vm.display.spice.streamingVideo.asString }"
<#if spec.vm.display.spice.streamingVideo??>
streaming-video: "${ spec.vm.display.spice.streamingVideo }"
</#if>
usbRedirects: ${ cr.spec.vm.display.spice.usbRedirects.asInt?c }
usbRedirects: ${ spec.vm.display.spice.usbRedirects?c }
</#if>
logging.properties: |

4
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDataPvc.ftl.yaml
View file

@ -1,11 +1,11 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
namespace: ${ cr.namespace() }
name: ${ runnerDataPvcName }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
spec:
accessModes:

8
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerDiskPvc.ftl.yaml
View file

@ -1,16 +1,16 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
namespace: ${ cr.namespace() }
name: ${ disk.generatedPvcName }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.annotations??>
annotations:
${ disk.volumeClaimTemplate.metadata.annotations.toString() }
${ toJson(disk.volumeClaimTemplate.metadata.annotations) }
</#if>
spec:
${ disk.volumeClaimTemplate.spec.toString() }
${ toJson(disk.volumeClaimTemplate.spec) }

16
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerLoadBalancer.ftl.yaml
View file

@ -1,26 +1,26 @@
apiVersion: v1
kind: Service
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
namespace: ${ cr.namespace() }
name: ${ cr.name() }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
vmoperator.jdrupes.org/version: ${ managerVersion }
ownerReferences:
- apiVersion: ${ cr.apiVersion.asString }
- apiVersion: ${ cr.apiVersion }
kind: ${ constants.VM_OP_KIND_VM }
name: ${ cr.metadata.name.asString }
uid: ${ cr.metadata.uid.asString }
name: ${ cr.name() }
uid: ${ cr.metadata().getUid() }
controller: false
spec:
type: LoadBalancer
ports:
- name: spice
port: ${ cr.spec.vm.display.spice.port.asInt?c }
port: ${ cr.spec().vm.display.spice.port?c }
selector:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }

73
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml
View file

@ -1,42 +1,43 @@
kind: Pod
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
namespace: ${ cr.namespace() }
name: ${ cr.name() }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/instance: ${ cr.name() }
app.kubernetes.io/component: ${ constants.APP_NAME }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
# Triggers update of config map mounted in pod
# See https://ahmet.im/blog/kubernetes-secret-volumes-delay/
vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }"
vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion }"
vmoperator.jdrupes.org/version: ${ managerVersion }
ownerReferences:
- apiVersion: ${ cr.apiVersion.asString }
- apiVersion: ${ cr.apiVersion }
kind: ${ constants.VM_OP_KIND_VM }
name: ${ cr.metadata.name.asString }
uid: ${ cr.metadata.uid.asString }
name: ${ cr.name() }
uid: ${ cr.metadata().getUid() }
blockOwnerDeletion: true
controller: false
<#assign spec = cr.spec() />
spec:
containers:
- name: ${ cr.metadata.name.asString }
<#assign image = cr.spec.image>
- name: ${ cr.name() }
<#assign image = spec.image>
<#if image.source??>
image: ${ image.source.asString }
image: ${ image.source }
<#else>
image: ${ image.repository.asString }/${ image.path.asString }<#if image.version??>:${ image.version.asString }</#if>
image: ${ image.repository }/${ image.path }<#if image.version??>:${ image.version }</#if>
</#if>
<#if image.pullPolicy??>
imagePullPolicy: ${ image.pullPolicy.asString }
imagePullPolicy: ${ image.pullPolicy }
</#if>
<#if cr.spec.vm.display.spice??>
<#if spec.vm.display.spice??>
ports:
<#if cr.spec.vm.display.spice??>
<#if spec.vm.display.spice??>
- name: spice
containerPort: ${ cr.spec.vm.display.spice.port.asInt?c }
containerPort: ${ spec.vm.display.spice.port?c }
protocol: TCP
</#if>
</#if>
@ -55,33 +56,33 @@ spec:
- name: vmop-image-repository
mountPath: ${ constants.IMAGE_REPO_PATH }
volumeDevices:
<#list cr.spec.vm.disks.asList() as disk>
<#list spec.vm.disks as disk>
<#if disk.volumeClaimTemplate??>
- name: ${ disk.generatedDiskName.asString }
devicePath: /dev/${ disk.generatedDiskName.asString }
- name: ${ disk.generatedDiskName }
devicePath: /dev/${ disk.generatedDiskName }
</#if>
</#list>
securityContext:
privileged: true
<#if cr.spec.resources??>
resources: ${ cr.spec.resources.toString() }
<#if spec.resources??>
resources: ${ toJson(spec.resources) }
<#else>
<#if cr.spec.vm.currentCpus?? || cr.spec.vm.currentRam?? >
<#if spec.vm.currentCpus?? || spec.vm.currentRam?? >
resources:
requests:
<#if cr.spec.vm.currentCpus?? >
<#if spec.vm.currentCpus?? >
<#assign factor = 2.0 />
<#if reconciler.cpuOvercommit??>
<#assign factor = reconciler.cpuOvercommit * 1.0 />
</#if>
cpu: ${ (parseQuantity(cr.spec.vm.currentCpus.asString) / factor)?c }
cpu: ${ (parseQuantity(spec.vm.currentCpus) / factor)?c }
</#if>
<#if cr.spec.vm.currentRam?? >
<#if spec.vm.currentRam?? >
<#assign factor = 1.25 />
<#if reconciler.ramOvercommit??>
<#assign factor = reconciler.ramOvercommit * 1.0 />
</#if>
memory: ${ (parseQuantity(cr.spec.vm.currentRam.asString) / factor)?floor?c }
memory: ${ (parseQuantity(spec.vm.currentRam) / factor)?floor?c }
</#if>
</#if>
</#if>
@ -102,7 +103,7 @@ spec:
projected:
sources:
- configMap:
name: ${ cr.metadata.name.asString }
name: ${ cr.name() }
<#if displaySecret??>
- secret:
name: ${ displaySecret }
@ -113,22 +114,22 @@ spec:
- name: runner-data
persistentVolumeClaim:
claimName: ${ runnerDataPvcName }
<#list cr.spec.vm.disks.asList() as disk>
<#list spec.vm.disks as disk>
<#if disk.volumeClaimTemplate??>
- name: ${ disk.generatedDiskName.asString }
- name: ${ disk.generatedDiskName }
persistentVolumeClaim:
claimName: ${ disk.generatedPvcName.asString }
claimName: ${ disk.generatedPvcName }
</#if>
</#list>
hostNetwork: true
terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
<#if cr.spec.nodeName??>
nodeName: ${ cr.spec.nodeName.asString }
terminationGracePeriodSeconds: ${ (spec.vm.powerdownTimeout + 5)?c }
<#if spec.nodeName??>
nodeName: ${ spec.nodeName }
</#if>
<#if cr.spec.nodeSelector??>
nodeSelector: ${ cr.spec.nodeSelector.toString() }
<#if spec.nodeSelector??>
nodeSelector: ${ toJson(spec.nodeSelector) }
</#if>
<#if cr.spec.affinity??>
affinity: ${ cr.spec.affinity.toString() }
<#if spec.affinity??>
affinity: ${ toJson(spec.affinity) }
</#if>
serviceAccountName: vm-runner

194
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerSts.ftl.yaml
View file

@ -1,194 +0,0 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
vmoperator.jdrupes.org/version: ${ managerVersion }
ownerReferences:
- apiVersion: ${ cr.apiVersion.asString }
kind: ${ constants.VM_OP_KIND_VM }
name: ${ cr.metadata.name.asString }
uid: ${ cr.metadata.uid.asString }
blockOwnerDeletion: true
controller: false
spec:
selector:
matchLabels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
replicas: ${ (cr.spec.vm.state.asString == "Running")?then(1, 0) }
updateStrategy:
type: OnDelete
template:
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/component: ${ constants.APP_NAME }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
annotations:
# Triggers update of config map mounted in pod
# See https://ahmet.im/blog/kubernetes-secret-volumes-delay/
vmrunner.jdrupes.org/cmVersion: "${ cm.metadata.resourceVersion.asString }"
vmoperator.jdrupes.org/version: ${ managerVersion }
spec:
containers:
- name: ${ cr.metadata.name.asString }
<#assign image = cr.spec.image>
<#if image.source??>
image: ${ image.source.asString }
<#else>
image: ${ image.repository.asString }/${ image.path.asString }<#if image.version??>:${ image.version.asString }</#if>
</#if>
<#if image.pullPolicy??>
imagePullPolicy: ${ image.pullPolicy.asString }
</#if>
<#if cr.spec.vm.display.spice??>
ports:
<#if cr.spec.vm.display.spice??>
- name: spice
containerPort: ${ cr.spec.vm.display.spice.port.asInt?c }
protocol: TCP
</#if>
</#if>
volumeMounts:
# Not needed because pod is priviledged:
# - mountPath: /dev/kvm
# name: dev-kvm
# - mountPath: /dev/net/tun
# name: dev-tun
# - mountPath: /sys/fs/cgroup
# name: cgroup
- name: config
mountPath: /etc/opt/vmrunner
- name: runner-data
mountPath: /var/local/vm-data
- name: vmop-image-repository
mountPath: ${ constants.IMAGE_REPO_PATH }
volumeDevices:
<#assign diskCounter = 0/>
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate??>
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.name??>
<#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
<#else>
<#assign diskName = "disk-" + diskCounter>
</#if>
- name: ${ diskName }
devicePath: /dev/${ diskName }
<#assign diskCounter = diskCounter + 1/>
</#if>
</#list>
securityContext:
privileged: true
<#if cr.spec.resources??>
resources: ${ cr.spec.resources.toString() }
<#else>
<#if cr.spec.vm.currentCpus?? || cr.spec.vm.currentRam?? >
resources:
requests:
<#if cr.spec.vm.currentCpus?? >
<#assign factor = 2.0 />
<#if reconciler.cpuOvercommit??>
<#assign factor = reconciler.cpuOvercommit * 1.0 />
</#if>
cpu: ${ (parseQuantity(cr.spec.vm.currentCpus.asString) / factor)?c }
</#if>
<#if cr.spec.vm.currentRam?? >
<#assign factor = 1.25 />
<#if reconciler.ramOvercommit??>
<#assign factor = reconciler.ramOvercommit * 1.0 />
</#if>
memory: ${ (parseQuantity(cr.spec.vm.currentRam.asString) / factor)?floor?c }
</#if>
</#if>
</#if>
volumes:
# Not needed because pod is priviledged:
# - name: dev-kvm
# hostPath:
# path: /dev/kvm
# type: CharDevice
# - hostPath:
# path: /dev/net/tun
# type: CharDevice
# name: dev-tun
# - name: cgroup
# hostPath:
# path: /sys/fs/cgroup
- name: config
projected:
sources:
- configMap:
name: ${ cr.metadata.name.asString }
<#if displaySecret??>
- secret:
name: ${ displaySecret }
</#if>
- name: vmop-image-repository
persistentVolumeClaim:
claimName: vmop-image-repository
hostNetwork: true
terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
<#if cr.spec.nodeName??>
nodeName: ${ cr.spec.nodeName.asString }
</#if>
<#if cr.spec.nodeSelector??>
nodeSelector: ${ cr.spec.nodeSelector.toString() }
</#if>
<#if cr.spec.affinity??>
affinity: ${ cr.spec.affinity.toString() }
</#if>
serviceAccountName: vm-runner
volumeClaimTemplates:
- metadata:
namespace: ${ cr.metadata.namespace.asString }
name: runner-data
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
spec:
accessModes:
- ReadWriteOnce
<#if reconciler.runnerDataPvc?? && reconciler.runnerDataPvc.storageClassName??>
storageClassName: ${ reconciler.runnerDataPvc.storageClassName }
</#if>
resources:
requests:
storage: 1Mi
<#assign diskCounter = 0/>
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate??>
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.name??>
<#assign diskName = disk.volumeClaimTemplate.metadata.name.asString + "-disk">
<#else>
<#assign diskName = "disk-" + diskCounter>
</#if>
- metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ diskName }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.annotations??>
annotations:
${ disk.volumeClaimTemplate.metadata.annotations.toString() }
</#if>
spec:
${ disk.volumeClaimTemplate.spec.toString() }
<#assign diskCounter = diskCounter + 1/>
</#if>
</#list>

7
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/ConfigMapReconciler.java
View file

@ -68,7 +68,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
* @throws TemplateException the template exception
* @throws ApiException the api exception
*/
public DynamicKubernetesObject reconcile(Map<String, Object> model,
public Map<String, Object> reconcile(Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
// Get API
@ -87,7 +87,10 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
// Apply and maybe force pod update
var newState = K8s.apply(cmApi, mapDef, out.toString());
maybeForceUpdate(channel.client(), newState);
return newState;
@SuppressWarnings("unchecked")
var res = (Map<String, Object>) channel.client().getJSON().getGson()
.fromJson(newState.getRaw(), Map.class);
return res;
}
/**

5
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretMonitor.java
View file

@ -256,10 +256,9 @@ public class DisplaySecretMonitor
@SuppressWarnings("PMD.AvoidSynchronizedStatement")
public void onVmDefChanged(VmDefChanged event, Channel channel) {
synchronized (pendingGets) {
String vmName = event.vmDefinition().metadata().getName();
String vmName = event.vmDefinition().name();
for (var pending : pendingGets) {
if (pending.event.vmDefinition().metadata().getName()
.equals(vmName)
if (pending.event.vmDefinition().name().equals(vmName)
&& event.vmDefinition().displayPasswordSerial()
.map(s -> s >= pending.expectedSerial).orElse(false)) {
pending.lock.remove();

24
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java
View file

@ -18,7 +18,6 @@
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonPrimitive;
import freemarker.template.TemplateException;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
@ -36,7 +35,7 @@ import static org.jdrupes.vmoperator.manager.Constants.DATA_DISPLAY_PASSWORD;
import static org.jdrupes.vmoperator.manager.Constants.DATA_PASSWORD_EXPIRY;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.jdrupes.vmoperator.util.DataPath;
import org.jose4j.base64url.Base64;
/**
@ -61,32 +60,31 @@ import org.jose4j.base64url.Base64;
Map<String, Object> model, VmChannel channel)
throws IOException, TemplateException, ApiException {
// Secret needed at all?
var display = GsonPtr.to(event.vmDefinition().data()).to("spec", "vm",
"display");
if (!display.get(JsonPrimitive.class, "spice", "generateSecret")
.map(JsonPrimitive::getAsBoolean).orElse(true)) {
var display = event.vmDefinition().fromVm("display").get();
if (!DataPath.<Boolean> get(display, "spice", "generateSecret")
.orElse(true)) {
return;
}
// Check if exists
var metadata = event.vmDefinition().getMetadata();
var vmDef = event.vmDefinition();
ListOptions options = new ListOptions();
options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + ","
+ "app.kubernetes.io/instance=" + metadata.getName());
var stubs = K8sV1SecretStub.list(channel.client(),
metadata.getNamespace(), options);
+ "app.kubernetes.io/instance=" + vmDef.name());
var stubs = K8sV1SecretStub.list(channel.client(), vmDef.namespace(),
options);
if (!stubs.isEmpty()) {
return;
}
// Create secret
var secret = new V1Secret();
secret.setMetadata(new V1ObjectMeta().namespace(metadata.getNamespace())
.name(metadata.getName() + "-" + COMP_DISPLAY_SECRET)
secret.setMetadata(new V1ObjectMeta().namespace(vmDef.namespace())
.name(vmDef.name() + "-" + COMP_DISPLAY_SECRET)
.putLabelsItem("app.kubernetes.io/name", APP_NAME)
.putLabelsItem("app.kubernetes.io/component", COMP_DISPLAY_SECRET)
.putLabelsItem("app.kubernetes.io/instance", metadata.getName()));
.putLabelsItem("app.kubernetes.io/instance", vmDef.name()));
secret.setType("Opaque");
SecureRandom random = null;
try {

104
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/LoadBalancerReconciler.java
View file

@ -18,22 +18,23 @@
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonObject;
import com.google.gson.Gson;
import freemarker.template.Configuration;
import freemarker.template.TemplateException;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1APIService;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Logger;
import org.jdrupes.vmoperator.common.K8s;
import org.jdrupes.vmoperator.common.K8sDynamicModel;
import org.jdrupes.vmoperator.common.K8sV1ServiceStub;
import org.jdrupes.vmoperator.common.VmDefinition;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
@ -92,11 +93,13 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
if (lbsDef instanceof Boolean isOn && !isOn) {
return;
}
JsonObject cfgMeta = new JsonObject();
if (lbsDef instanceof Map) {
var json = channel.client().getJSON();
cfgMeta
= json.deserialize(json.serialize(lbsDef), JsonObject.class);
// Load balancer can also be turned off for VM
var vmDef = event.vmDefinition();
if (vmDef
.<Map<String, Map<String, String>>> fromSpec(LOAD_BALANCER_SERVICE)
.map(m -> m.isEmpty()).orElse(false)) {
return;
}
// Combine template and data and parse result
@ -107,53 +110,78 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
// https://github.com/kubernetes-client/java/issues/2741
var svcDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
mergeMetadata(svcDef, cfgMeta, event.vmDefinition());
@SuppressWarnings("unchecked")
var defaults = lbsDef instanceof Map
? (Map<String, Map<String, String>>) lbsDef
: null;
var client = channel.client();
mergeMetadata(client.getJSON().getGson(), svcDef, defaults, vmDef);
// Apply
DynamicKubernetesApi svcApi = new DynamicKubernetesApi("", "v1",
"services", channel.client());
K8s.apply(svcApi, svcDef, svcDef.getRaw().toString());
var svcStub = K8sV1ServiceStub
.get(client, vmDef.namespace(), vmDef.name());
if (svcStub.apply(svcDef).isEmpty()) {
logger.warning(
() -> "Could not patch service for " + svcStub.name());
}
}
private void mergeMetadata(DynamicKubernetesObject svcDef,
JsonObject cfgMeta, K8sDynamicModel vmDefinition) {
// Get metadata from VM definition
var vmMeta = GsonPtr.to(vmDefinition.data()).to("spec")
.get(JsonObject.class, LOAD_BALANCER_SERVICE)
.map(JsonObject::deepCopy).orElseGet(() -> new JsonObject());
private void mergeMetadata(Gson gson, DynamicKubernetesObject svcDef,
Map<String, Map<String, String>> defaults,
VmDefinition vmDefinition) {
// Get specific load balancer metadata from VM definition
var vmLbMeta = vmDefinition
.<Map<String, Map<String, String>>> fromSpec(LOAD_BALANCER_SERVICE)
.orElse(Collections.emptyMap());
// Merge Data from VM definition into config data
mergeReplace(GsonPtr.to(cfgMeta).to(LABELS).get(JsonObject.class),
GsonPtr.to(vmMeta).to(LABELS).get(JsonObject.class));
mergeReplace(
GsonPtr.to(cfgMeta).to(ANNOTATIONS).get(JsonObject.class),
GsonPtr.to(vmMeta).to(ANNOTATIONS).get(JsonObject.class));
// Merge additional data into service definition
var svcMeta = GsonPtr.to(svcDef.getRaw()).to(METADATA);
mergeIfAbsent(svcMeta.to(LABELS).get(JsonObject.class),
GsonPtr.to(cfgMeta).to(LABELS).get(JsonObject.class));
mergeIfAbsent(svcMeta.to(ANNOTATIONS).get(JsonObject.class),
GsonPtr.to(cfgMeta).to(ANNOTATIONS).get(JsonObject.class));
// Merge
var svcMeta = svcDef.getMetadata();
var svcJsonMeta = GsonPtr.to(svcDef.getRaw()).to(METADATA);
Optional.ofNullable(mergeIfAbsent(svcMeta.getLabels(),
mergeReplace(defaults.get(LABELS), vmLbMeta.get(LABELS))))
.ifPresent(lbls -> svcJsonMeta.set(LABELS, gson.toJsonTree(lbls)));
Optional.ofNullable(mergeIfAbsent(svcMeta.getAnnotations(),
mergeReplace(defaults.get(ANNOTATIONS), vmLbMeta.get(ANNOTATIONS))))
.ifPresent(as -> svcJsonMeta.set(ANNOTATIONS, gson.toJsonTree(as)));
}
private void mergeReplace(JsonObject dest, JsonObject src) {
private Map<String, String> mergeReplace(Map<String, String> dest,
Map<String, String> src) {
if (src == null) {
return dest;
}
if (dest == null) {
dest = new LinkedHashMap<>();
} else {
dest = new LinkedHashMap<>(dest);
}
for (var e : src.entrySet()) {
if (e.getValue().isJsonNull()) {
if (e.getValue() == null) {
dest.remove(e.getKey());
continue;
}
dest.add(e.getKey(), e.getValue());
dest.put(e.getKey(), e.getValue());
}
return dest;
}
private void mergeIfAbsent(JsonObject dest, JsonObject src) {
private Map<String, String> mergeIfAbsent(Map<String, String> dest,
Map<String, String> src) {
if (src == null) {
return dest;
}
if (dest == null) {
dest = new LinkedHashMap<>();
} else {
dest = new LinkedHashMap<>(dest);
}
for (var e : src.entrySet()) {
if (dest.has(e.getKey())) {
if (dest.containsKey(e.getKey())) {
continue;
}
dest.add(e.getKey(), e.getValue());
dest.put(e.getKey(), e.getValue());
}
return dest;
}
}

17
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PodReconciler.java
View file

@ -20,7 +20,6 @@ package org.jdrupes.vmoperator.manager;
import freemarker.template.Configuration;
import freemarker.template.TemplateException;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import io.kubernetes.client.util.generic.options.PatchOptions;
@ -29,7 +28,7 @@ import java.io.StringWriter;
import java.util.Map;
import java.util.logging.Logger;
import org.jdrupes.vmoperator.common.K8sV1PodStub;
import org.jdrupes.vmoperator.common.VmDefinitionModel.RequestedVmState;
import org.jdrupes.vmoperator.common.VmDefinition.RequestedVmState;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.yaml.snakeyaml.LoaderOptions;
@ -73,18 +72,18 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
}
// Get pod stub.
var metadata = event.vmDefinition().getMetadata();
var podStub = K8sV1PodStub.get(channel.client(),
metadata.getNamespace(), metadata.getName());
var vmDef = event.vmDefinition();
var podStub = K8sV1PodStub.get(channel.client(), vmDef.namespace(),
vmDef.name());
// Nothing to do if exists and should be running
if (event.vmDefinition().vmState() == RequestedVmState.RUNNING
if (vmDef.vmState() == RequestedVmState.RUNNING
&& podStub.model().isPresent()) {
return;
}
// Delete if running but should be stopped
if (event.vmDefinition().vmState() == RequestedVmState.STOPPED) {
if (vmDef.vmState() == RequestedVmState.STOPPED) {
if (podStub.model().isPresent()) {
podStub.delete();
}
@ -104,9 +103,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
if (podStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(podDef)), opts)
.isEmpty()) {
if (podStub.apply(podDef).isEmpty()) {
logger.warning(
() -> "Could not patch pod for " + podStub.name());
}

52
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/PvcReconciler.java
View file

@ -18,8 +18,6 @@
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import freemarker.core.ParseException;
import freemarker.template.Configuration;
import freemarker.template.MalformedTemplateNameException;
@ -32,6 +30,7 @@ import io.kubernetes.client.util.generic.options.ListOptions;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
@ -41,7 +40,7 @@ import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME;
import org.jdrupes.vmoperator.common.K8sV1PvcStub;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.jdrupes.vmoperator.util.DataPath;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
@ -78,16 +77,16 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
public void reconcile(VmDefChanged event, Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
var vmDef = event.vmDefinition();
// Existing disks
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector(
"app.kubernetes.io/managed-by=" + VM_OP_NAME + ","
+ "app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=" + metadata.getName());
+ "app.kubernetes.io/instance=" + vmDef.name());
var knownDisks = K8sV1PvcStub.list(channel.client(),
metadata.getNamespace(), listOpts);
vmDef.namespace(), listOpts);
var knownPvcs = knownDisks.stream().map(K8sV1PvcStub::name)
.collect(Collectors.toSet());
@ -95,23 +94,23 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
reconcileRunnerDataPvc(event, model, channel, knownPvcs);
// Reconcile pvcs for defined disks
var diskDefs = GsonPtr.to((JsonObject) model.get("cr"))
.getAsListOf(JsonObject.class, "spec", "vm", "disks");
var diskDefs = vmDef.<List<Map<String, Object>>> fromVm("disks")
.orElse(List.of());
var diskCounter = 0;
for (var diskDef : diskDefs) {
if (!diskDef.has("volumeClaimTemplate")) {
if (!diskDef.containsKey("volumeClaimTemplate")) {
continue;
}
var diskName = GsonPtr.to(diskDef)
.getAsString("volumeClaimTemplate", "metadata", "name")
.map(name -> name + "-disk").orElse("disk-" + diskCounter);
var diskName = DataPath.get(diskDef, "volumeClaimTemplate",
"metadata", "name").map(name -> name + "-disk")
.orElse("disk-" + diskCounter);
diskCounter += 1;
diskDef.addProperty("generatedDiskName", diskName);
diskDef.put("generatedDiskName", diskName);
// Don't do anything if pvc with old (sts generated) name exists.
var stsDiskPvcName = diskName + "-" + metadata.getName() + "-0";
var stsDiskPvcName = diskName + "-" + vmDef.name() + "-0";
if (knownPvcs.contains(stsDiskPvcName)) {
diskDef.addProperty("generatedPvcName", stsDiskPvcName);
diskDef.put("generatedPvcName", stsDiskPvcName);
continue;
}
@ -127,18 +126,18 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
Set<String> knownPvcs)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
var vmDef = event.vmDefinition();
// Look for old (sts generated) name.
var stsRunnerDataPvcName
= "runner-data" + "-" + metadata.getName() + "-0";
= "runner-data" + "-" + vmDef.name() + "-0";
if (knownPvcs.contains(stsRunnerDataPvcName)) {
model.put("runnerDataPvcName", stsRunnerDataPvcName);
return;
}
// Generate PVC
model.put("runnerDataPvcName", metadata.getName() + "-runner-data");
model.put("runnerDataPvcName", vmDef.name() + "-runner-data");
var fmTemplate = fmConfig.getTemplate("runnerDataPvc.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
@ -149,7 +148,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
// Do apply changes
var pvcStub = K8sV1PvcStub.get(channel.client(),
metadata.getNamespace(), (String) model.get("runnerDataPvcName"));
vmDef.namespace(), (String) model.get("runnerDataPvcName"));
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
@ -165,13 +164,13 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
Map<String, Object> model, VmChannel channel)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
var vmDef = event.vmDefinition();
// Generate PVC
var diskDef = GsonPtr.to((JsonElement) model.get("disk"));
var pvcName = metadata.getName() + "-"
+ diskDef.getAsString("generatedDiskName").get();
diskDef.set("generatedPvcName", pvcName);
@SuppressWarnings("unchecked")
var diskDef = (Map<String, Object>) model.get("disk");
var pvcName = vmDef.name() + "-" + diskDef.get("generatedDiskName");
diskDef.put("generatedPvcName", pvcName);
var fmTemplate = fmConfig.getTemplate("runnerDiskPvc.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
@ -181,9 +180,8 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
// Do apply changes
var pvcStub = K8sV1PvcStub.get(channel.client(),
metadata.getNamespace(), GsonPtr.to((JsonElement) model.get("disk"))
.getAsString("generatedPvcName").get());
var pvcStub
= K8sV1PvcStub.get(channel.client(), vmDef.namespace(), pvcName);
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");

193
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java
View file

@ -18,11 +18,13 @@
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import freemarker.template.AdapterTemplateModel;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapperBuilder;
import freemarker.template.SimpleNumber;
import freemarker.template.SimpleScalar;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import freemarker.template.TemplateHashModel;
@ -30,7 +32,7 @@ import freemarker.template.TemplateMethodModelEx;
import freemarker.template.TemplateModelException;
import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.util.generic.options.ListOptions;
import java.io.IOException;
import java.math.BigDecimal;
@ -44,15 +46,15 @@ import java.util.Optional;
import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
import org.jdrupes.vmoperator.common.Convertions;
import org.jdrupes.vmoperator.common.K8sClient;
import org.jdrupes.vmoperator.common.K8sDynamicModel;
import org.jdrupes.vmoperator.common.K8sObserver;
import org.jdrupes.vmoperator.common.K8sV1SecretStub;
import org.jdrupes.vmoperator.common.VmDefinition;
import static org.jdrupes.vmoperator.manager.Constants.COMP_DISPLAY_SECRET;
import org.jdrupes.vmoperator.manager.events.ResetVm;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.DataPath;
import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.jgrapes.core.Channel;
import org.jgrapes.core.Component;
import org.jgrapes.core.annotation.Handler;
@ -135,6 +137,10 @@ import org.jgrapes.util.events.ConfigurationUpdate;
"PMD.AvoidDuplicateLiterals" })
public class Reconciler extends Component {
/** The Constant mapper. */
@SuppressWarnings("PMD.FieldNamingConventions")
protected static final ObjectMapper mapper = new ObjectMapper();
@SuppressWarnings("PMD.SingularField")
private final Configuration fmConfig;
private final ConfigMapReconciler cmReconciler;
@ -203,17 +209,17 @@ public class Reconciler extends Component {
}
// Ownership relationships takes care of deletions
var defMeta = event.vmDefinition().getMetadata();
if (event.type() == K8sObserver.ResponseType.DELETED) {
logger.fine(() -> "VM \"" + defMeta.getName() + "\" deleted");
logger.fine(
() -> "VM \"" + event.vmDefinition().name() + "\" deleted");
return;
}
// Reconcile, use "augmented" vm definition for model
// Create model for processing templates
Map<String, Object> model
= prepareModel(channel.client(), patchCr(event.vmDefinition()));
= prepareModel(channel.client(), event.vmDefinition());
var configMap = cmReconciler.reconcile(model, channel);
model.put("cm", configMap.getRaw());
model.put("cm", configMap);
dsReconciler.reconcile(event, model, channel);
// Manage (eventual) removal of stateful set.
stsReconciler.reconcile(event, model, channel);
@ -235,81 +241,22 @@ public class Reconciler extends Component {
@Handler
public void onResetVm(ResetVm event, VmChannel channel)
throws ApiException, IOException, TemplateException {
var defRoot
= GsonPtr.to(channel.vmDefinition().data()).get(JsonObject.class);
defRoot.addProperty("resetCount",
defRoot.get("resetCount").getAsLong() + 1);
var vmDef = channel.vmDefinition();
vmDef.extra("resetCount", vmDef.<Long> extra("resetCount") + 1);
Map<String, Object> model
= prepareModel(channel.client(), patchCr(channel.vmDefinition()));
= prepareModel(channel.client(), channel.vmDefinition());
cmReconciler.reconcile(model, channel);
}
private DynamicKubernetesObject patchCr(K8sDynamicModel vmDef) {
var json = vmDef.data().deepCopy();
// Adjust cdromImage path
adjustCdRomPaths(json);
// Adjust cloud-init data
adjustCloudInitData(json);
return new DynamicKubernetesObject(json);
}
@SuppressWarnings("PMD.AvoidLiteralsInIfCondition")
private void adjustCdRomPaths(JsonObject json) {
var disks
= GsonPtr.to(json).to("spec", "vm", "disks").get(JsonArray.class);
for (var disk : disks) {
var cdrom = (JsonObject) ((JsonObject) disk).get("cdrom");
if (cdrom == null) {
continue;
}
String image = cdrom.get("image").getAsString();
if (image.isEmpty()) {
continue;
}
try {
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH
+ "/").resolve(image);
if ("file".equals(imageUri.getScheme())) {
cdrom.addProperty("image", imageUri.getPath());
} else {
cdrom.addProperty("image", imageUri.toString());
}
} catch (URISyntaxException e) {
logger.warning(() -> "Invalid CDROM image: " + image);
}
}
}
private void adjustCloudInitData(JsonObject json) {
var spec = GsonPtr.to(json).to("spec").get(JsonObject.class);
if (!spec.has("cloudInit")) {
return;
}
var metaData = GsonPtr.to(spec).to("cloudInit", "metaData");
if (metaData.getAsString("instance-id").isEmpty()) {
metaData.set("instance-id",
GsonPtr.to(json).getAsString("metadata", "resourceVersion")
.map(s -> "v" + s).orElse("v1"));
}
if (metaData.getAsString("local-hostname").isEmpty()) {
metaData.set("local-hostname",
GsonPtr.to(json).getAsString("metadata", "name").get());
}
}
@SuppressWarnings("PMD.CognitiveComplexity")
@SuppressWarnings({ "PMD.CognitiveComplexity", "PMD.NPathComplexity" })
private Map<String, Object> prepareModel(K8sClient client,
DynamicKubernetesObject vmDef)
throws TemplateModelException, ApiException {
VmDefinition vmDef) throws TemplateModelException, ApiException {
@SuppressWarnings("PMD.UseConcurrentHashMap")
Map<String, Object> model = new HashMap<>();
model.put("managerVersion",
Optional.ofNullable(Reconciler.class.getPackage()
.getImplementationVersion()).orElse("(Unknown)"));
model.put("cr", vmDef.getRaw());
model.put("cr", vmDef);
model.put("constants",
(TemplateHashModel) new DefaultObjectWrapperBuilder(
Configuration.VERSION_2_3_32)
@ -321,9 +268,10 @@ public class Reconciler extends Component {
ListOptions options = new ListOptions();
options.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET + ","
+ "app.kubernetes.io/instance=" + vmDef.getMetadata().getName());
+ "app.kubernetes.io/instance=" + vmDef.name());
var dsStub = K8sV1SecretStub
.list(client, vmDef.getMetadata().getNamespace(), options).stream()
.list(client, vmDef.namespace(), options)
.stream()
.findFirst();
if (dsStub.isPresent()) {
dsStub.get().model().ifPresent(m -> {
@ -332,14 +280,23 @@ public class Reconciler extends Component {
}
// Methods
model.put("parseQuantity", new TemplateMethodModelEx() {
model.put("parseQuantity", parseQuantityModel);
model.put("formatMemory", formatMemoryModel);
model.put("imageLocation", imgageLocationModel);
model.put("adjustCloudInitMeta", adjustCloudInitMetaModel);
model.put("toJson", toJsonModel);
return model;
}
private final TemplateMethodModelEx parseQuantityModel
= new TemplateMethodModelEx() {
@Override
@SuppressWarnings("PMD.PreserveStackTrace")
public Object exec(@SuppressWarnings("rawtypes") List arguments)
throws TemplateModelException {
var arg = arguments.get(0);
if (arg instanceof Number number) {
return number;
if (arg instanceof SimpleNumber number) {
return number.getAsNumber();
}
try {
return Quantity.fromString(arg.toString()).getNumber();
@ -348,8 +305,10 @@ public class Reconciler extends Component {
+ "specified as \"" + arg + "\": " + e.getMessage());
}
}
});
model.put("formatMemory", new TemplateMethodModelEx() {
};
private final TemplateMethodModelEx formatMemoryModel
= new TemplateMethodModelEx() {
@Override
@SuppressWarnings("PMD.PreserveStackTrace")
public Object exec(@SuppressWarnings("rawtypes") List arguments)
@ -376,7 +335,71 @@ public class Reconciler extends Component {
}
return Convertions.formatMemory(bigInt);
}
});
return model;
}
};
private final TemplateMethodModelEx imgageLocationModel
= new TemplateMethodModelEx() {
@Override
@SuppressWarnings({ "PMD.PreserveStackTrace",
"PMD.AvoidLiteralsInIfCondition" })
public Object exec(@SuppressWarnings("rawtypes") List arguments)
throws TemplateModelException {
var image = ((SimpleScalar) arguments.get(0)).getAsString();
if (image.isEmpty()) {
return "";
}
try {
var imageUri = new URI("file://" + Constants.IMAGE_REPO_PATH
+ "/").resolve(image);
if ("file".equals(imageUri.getScheme())) {
return imageUri.getPath();
}
return imageUri.toString();
} catch (URISyntaxException e) {
logger.warning(() -> "Invalid CDROM image: " + image);
}
return image;
}
};
private final TemplateMethodModelEx adjustCloudInitMetaModel
= new TemplateMethodModelEx() {
@Override
@SuppressWarnings("PMD.PreserveStackTrace")
public Object exec(@SuppressWarnings("rawtypes") List arguments)
throws TemplateModelException {
@SuppressWarnings("unchecked")
var res = (Map<String, Object>) DataPath
.deepCopy(((AdapterTemplateModel) arguments.get(0))
.getAdaptedObject(Object.class));
var metadata
= (V1ObjectMeta) ((AdapterTemplateModel) arguments.get(1))
.getAdaptedObject(Object.class);
if (!res.containsKey("instance-id")) {
res.put("instance-id",
Optional.ofNullable(metadata.getResourceVersion())
.map(s -> "v" + s).orElse("v1"));
}
if (!res.containsKey("local-hostname")) {
res.put("local-hostname", metadata.getName());
}
return res;
}
};
private final TemplateMethodModelEx toJsonModel
= new TemplateMethodModelEx() {
@Override
@SuppressWarnings("PMD.PreserveStackTrace")
public Object exec(@SuppressWarnings("rawtypes") List arguments)
throws TemplateModelException {
try {
return mapper.writeValueAsString(
((AdapterTemplateModel) arguments.get(0))
.getAdaptedObject(Object.class));
} catch (JsonProcessingException e) {
return "{}";
}
}
};
}

33
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/StatefulSetReconciler.java
View file

@ -22,19 +22,14 @@ import freemarker.template.Configuration;
import freemarker.template.TemplateException;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import java.util.logging.Logger;
import org.jdrupes.vmoperator.common.K8sV1StatefulSetStub;
import org.jdrupes.vmoperator.common.VmDefinition.RequestedVmState;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.yaml.snakeyaml.LoaderOptions;
import org.yaml.snakeyaml.Yaml;
import org.yaml.snakeyaml.constructor.SafeConstructor;
/**
* Before version 3.4, the pod running the VM was created by a stateful set.
@ -45,15 +40,15 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
/* default */ class StatefulSetReconciler {
protected final Logger logger = Logger.getLogger(getClass().getName());
private final Configuration fmConfig;
/**
* Instantiates a new stateful set reconciler.
*
* @param fmConfig the fm config
*/
@SuppressWarnings("PMD.UnusedFormalParameter")
public StatefulSetReconciler(Configuration fmConfig) {
this.fmConfig = fmConfig;
// Nothing to do
}
/**
@ -70,12 +65,11 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
public void reconcile(VmDefChanged event, Map<String, Object> model,
VmChannel channel)
throws IOException, TemplateException, ApiException {
var metadata = event.vmDefinition().getMetadata();
model.put("usingSts", false);
// If exists, delete when not running or supposed to be not running.
var stsStub = K8sV1StatefulSetStub.get(channel.client(),
metadata.getNamespace(), metadata.getName());
event.vmDefinition().namespace(), event.vmDefinition().name());
if (stsStub.model().isEmpty()) {
return;
}
@ -94,16 +88,7 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
// Check if VM is supposed to be stopped. If so,
// set replicas to 0. This is the first step of the transition,
// the stateful set will be deleted when the VM is restarted.
var fmTemplate = fmConfig.getTemplate("runnerSts.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var stsDef = Dynamics.newFromYaml(
new Yaml(new SafeConstructor(new LoaderOptions())), out.toString());
var desired = GsonPtr.to(stsDef.getRaw())
.to("spec").getAsInt("replicas").orElse(1);
if (desired == 1) {
if (event.vmDefinition().vmState() == RequestedVmState.RUNNING) {
return;
}
@ -111,12 +96,12 @@ import org.yaml.snakeyaml.constructor.SafeConstructor;
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
if (stsStub.patch(V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(stsDef)), opts)
.isEmpty()) {
if (stsStub.patch(V1Patch.PATCH_FORMAT_JSON_PATCH,
new V1Patch("[{\"op\": \"replace\", \"path\": \"/spec/replicas"
+ "\", \"value\": 0}]"),
channel.client().defaultPatchOptions()).isEmpty()) {
logger.warning(
() -> "Could not patch stateful set for " + stsStub.name());
}
}
}

90
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java
View file

@ -18,13 +18,15 @@
package org.jdrupes.vmoperator.manager;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.util.Watch;
import io.kubernetes.client.util.generic.options.ListOptions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
@ -37,6 +39,7 @@ import org.jdrupes.vmoperator.common.K8sObserver.ResponseType;
import org.jdrupes.vmoperator.common.K8sV1ConfigMapStub;
import org.jdrupes.vmoperator.common.K8sV1PodStub;
import org.jdrupes.vmoperator.common.K8sV1StatefulSetStub;
import org.jdrupes.vmoperator.common.VmDefinition;
import org.jdrupes.vmoperator.common.VmDefinitionModel;
import org.jdrupes.vmoperator.common.VmDefinitionModels;
import org.jdrupes.vmoperator.common.VmDefinitionStub;
@ -46,7 +49,7 @@ import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME;
import org.jdrupes.vmoperator.manager.events.ChannelManager;
import org.jdrupes.vmoperator.manager.events.VmChannel;
import org.jdrupes.vmoperator.manager.events.VmDefChanged;
import org.jdrupes.vmoperator.util.GsonPtr;
import org.jdrupes.vmoperator.util.DataPath;
import org.jgrapes.core.Channel;
import org.jgrapes.core.Event;
@ -116,42 +119,47 @@ public class VmMonitor extends
V1ObjectMeta metadata = response.object.getMetadata();
VmChannel channel = channelManager.channelGet(metadata.getName());
// Remove from channel manager if deleted
if (ResponseType.valueOf(response.type) == ResponseType.DELETED) {
channelManager.remove(metadata.getName());
}
// Get full definition and associate with channel as backup
var vmDef = response.object;
if (vmDef.data() == null) {
var vmModel = response.object;
if (vmModel.data() == null) {
// ADDED event does not provide data, see
// https://github.com/kubernetes-client/java/issues/3215
vmDef = getModel(client, vmDef);
vmModel = getModel(client, vmModel);
}
if (vmDef.data() != null) {
VmDefinition vmDef = null;
if (vmModel.data() != null) {
// New data, augment and save
vmDef = client.getJSON().getGson().fromJson(vmModel.data(),
VmDefinition.class);
addDynamicData(channel.client(), vmDef, channel.vmDefinition());
channel.setVmDefinition(vmDef);
} else {
// Reuse cached
}
if (vmDef == null) {
// Reuse cached (e.g. if deleted)
vmDef = channel.vmDefinition();
}
if (vmDef == null) {
logger.warning(
() -> "Cannot get model for " + response.object.getMetadata());
logger.warning(() -> "Cannot get defintion for "
+ response.object.getMetadata());
return;
}
if (ResponseType.valueOf(response.type) == ResponseType.DELETED) {
channelManager.remove(metadata.getName());
}
// Create and fire changed event. Remove channel from channel
// manager on completion.
channel.pipeline()
.fire(Event.onCompletion(
new VmDefChanged(ResponseType.valueOf(response.type),
channel.setGeneration(
response.object.getMetadata().getGeneration()),
channel.setGeneration(response.object.getMetadata()
.getGeneration()),
vmDef),
e -> {
if (e.type() == ResponseType.DELETED) {
channelManager
.remove(e.vmDefinition().metadata().getName());
channelManager.remove(e.vmDefinition().name());
}
}), channel);
}
@ -166,51 +174,53 @@ public class VmMonitor extends
}
}
private void addDynamicData(K8sClient client, VmDefinitionModel vmState,
VmDefinitionModel prevState) {
var rootNode = GsonPtr.to(vmState.data()).get(JsonObject.class);
@SuppressWarnings("PMD.AvoidDuplicateLiterals")
private void addDynamicData(K8sClient client, VmDefinition vmDef,
VmDefinition prevState) {
// Maintain (or initialize) the resetCount
rootNode.addProperty("resetCount", Optional.ofNullable(prevState)
.map(ps -> GsonPtr.to(ps.data()))
.flatMap(d -> d.getAsLong("resetCount")).orElse(0L));
vmDef.extra("resetCount",
Optional.ofNullable(prevState).map(d -> d.extra("resetCount"))
.orElse(0L));
// Node information
// Add defaults in case the VM is not running
rootNode.addProperty("nodeName", "");
rootNode.addProperty("nodeAddress", "");
vmDef.extra("nodeName", "");
vmDef.extra("nodeAddress", "");
// VM definition status changes before the pod terminates.
// This results in pod information being shown for a stopped
// VM which is irritating. So check condition first.
var isRunning = GsonPtr.to(rootNode).to("status", "conditions")
.get(JsonArray.class)
.asList().stream().filter(el -> "Running"
.equals(((JsonObject) el).get("type").getAsString()))
.findFirst().map(el -> "True"
.equals(((JsonObject) el).get("status").getAsString()))
.orElse(false);
@SuppressWarnings("PMD.LambdaCanBeMethodReference")
var isRunning
= vmDef.<List<Map<String, Object>>> fromStatus("conditions")
.orElse(Collections.emptyList()).stream()
.filter(cond -> DataPath.get(cond, "type")
.map(t -> "Running".equals(t)).orElse(false))
.findFirst().map(cond -> DataPath.get(cond, "status")
.map(s -> "True".equals(s)).orElse(false))
.orElse(false);
if (!isRunning) {
return;
}
var podSearch = new ListOptions();
podSearch.setLabelSelector("app.kubernetes.io/name=" + APP_NAME
+ ",app.kubernetes.io/component=" + APP_NAME
+ ",app.kubernetes.io/instance=" + vmState.getMetadata().getName());
+ ",app.kubernetes.io/instance=" + vmDef.name());
try {
var podList
= K8sV1PodStub.list(client, namespace(), podSearch);
for (var podStub : podList) {
var nodeName = podStub.model().get().getSpec().getNodeName();
rootNode.addProperty("nodeName", nodeName);
vmDef.extra("nodeName", nodeName);
logger.fine(() -> "Added node name " + nodeName
+ " to VM info for " + vmState.getMetadata().getName());
+ " to VM info for " + vmDef.name());
@SuppressWarnings("PMD.AvoidInstantiatingObjectsInLoops")
var addrs = new JsonArray();
var addrs = new ArrayList<String>();
podStub.model().get().getStatus().getPodIPs().stream()
.map(ip -> ip.getIp()).forEach(addrs::add);
rootNode.add("nodeAddresses", addrs);
vmDef.extra("nodeAddresses", addrs);
logger.fine(() -> "Added node addresses " + addrs
+ " to VM info for " + vmState.getMetadata().getName());
+ " to VM info for " + vmDef.name());
}
} catch (ApiException e) {
logger.log(Level.WARNING, e,

64
org.jdrupes.vmoperator.manager/test-resources/basic-vm.yaml Normal file
View file

@ -0,0 +1,64 @@
apiVersion: "vmoperator.jdrupes.org/v1"
kind: VirtualMachine
metadata:
namespace: vmop-dev
name: unittest-vm
spec:
image:
repository: docker-registry.lan.mnl.de
path: vmoperator/this.will.never.start
version: 0.0.0
cloudInit:
metaData: {}
vm:
# state: Running
maximumRam: 4Gi
currentRam: 2Gi
maximumCpus: 4
currentCpus: 2
powerdownTimeout: 1
networks:
- user: {}
disks:
- cdrom:
image: https://test.com/test.iso
bootindex: 0
- cdrom:
image: "image.iso"
- volumeClaimTemplate:
metadata:
name: system
annotations:
use_as: system-disk
spec:
storageClassName: local-path
resources:
requests:
storage: 1Gi
- volumeClaimTemplate:
spec:
storageClassName: local-path
resources:
requests:
storage: 1Gi
display:
outputs: 2
spice:
port: 5812
usbRedirects: 2
resources:
requests:
cpu: 1
memory: 2Gi
loadBalancerService:
labels:
label2: replaced
label3: added
annotations:
anno1: added

35
org.jdrupes.vmoperator.manager/test-resources/unittest-vm.yaml
View file

@ -1,35 +0,0 @@
apiVersion: "vmoperator.jdrupes.org/v1"
kind: VirtualMachine
metadata:
namespace: vmop-dev
name: unittest-vm
spec:
resources:
requests:
cpu: 1
memory: 2Gi
loadBalancerService:
labels:
test2: null
test3: added
vm:
# state: Running
maximumRam: 4Gi
currentRam: 2Gi
maximumCpus: 4
currentCpus: 2
powerdownTimeout: 1
networks:
- user: {}
disks:
- cdrom:
# image: ""
image: https://download.fedoraproject.org/pub/fedora/linux/releases/38/Workstation/x86_64/iso/Fedora-Workstation-Live-x86_64-38-1.6.iso
# image: "Fedora-Workstation-Live-x86_64-38-1.6.iso"
display:
spice:
port: 5812

319
org.jdrupes.vmoperator.manager/test/org/jdrupes/vmoperator/manager/BasicTests.java
View file

@ -1,12 +1,19 @@
package org.jdrupes.vmoperator.manager;
import io.kubernetes.client.Discovery.APIResource;
import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.options.ListOptions;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.jdrupes.vmoperator.common.Constants.APP_NAME;
import static org.jdrupes.vmoperator.common.Constants.COMP_DISPLAY_SECRET;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_GROUP;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_KIND_VM;
import static org.jdrupes.vmoperator.common.Constants.VM_OP_NAME;
@ -15,7 +22,11 @@ import org.jdrupes.vmoperator.common.K8sClient;
import org.jdrupes.vmoperator.common.K8sDynamicStub;
import org.jdrupes.vmoperator.common.K8sV1ConfigMapStub;
import org.jdrupes.vmoperator.common.K8sV1DeploymentStub;
import org.jdrupes.vmoperator.common.K8sV1PodStub;
import org.jdrupes.vmoperator.common.K8sV1PvcStub;
import org.jdrupes.vmoperator.common.K8sV1SecretStub;
import org.jdrupes.vmoperator.common.K8sV1ServiceStub;
import org.jdrupes.vmoperator.util.DataPath;
import org.junit.jupiter.api.AfterAll;
import static org.junit.jupiter.api.Assertions.*;
import org.junit.jupiter.api.BeforeAll;
@ -29,6 +40,9 @@ class BasicTests {
private static K8sClient client;
private static APIResource vmsContext;
private static K8sV1DeploymentStub mgrDeployment;
private static K8sDynamicStub vmStub;
private static final String VM_NAME = "unittest-vm";
private static final Object EXISTS = new Object();
@BeforeAll
static void setUpBeforeClass() throws Exception {
@ -38,23 +52,40 @@ class BasicTests {
// Get client
client = new K8sClient();
// Update manager pod by scaling deployment
mgrDeployment
= K8sV1DeploymentStub.get(client, "vmop-dev", "vm-operator");
mgrDeployment.scale(0);
mgrDeployment.scale(1);
waitForManager();
// Context for working with our CR
var apiRes = K8s.context(client, VM_OP_GROUP, null, VM_OP_KIND_VM);
assertTrue(apiRes.isPresent());
vmsContext = apiRes.get();
// Cleanup existing VM
K8sDynamicStub.get(client, vmsContext, "vmop-dev", "unittest-vm")
K8sDynamicStub.get(client, vmsContext, "vmop-dev", VM_NAME)
.delete();
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=" + VM_NAME + ","
+ "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET);
var secrets = K8sV1SecretStub.list(client, "vmop-dev", listOpts);
for (var secret : secrets) {
secret.delete();
}
deletePvcs();
// Update manager pod by scaling deployment
mgrDeployment
= K8sV1DeploymentStub.get(client, "vmop-dev", "vm-operator");
mgrDeployment.scale(0);
mgrDeployment.scale(1);
// Load from Yaml
var rdr = new FileReader("test-resources/basic-vm.yaml");
vmStub = K8sDynamicStub.createFromYaml(client, vmsContext, rdr);
assertTrue(vmStub.model().isPresent());
}
private static void waitForManager()
throws ApiException, InterruptedException {
// Wait until available
for (int i = 0; i < 10; i++) {
if (mgrDeployment.model().get().getStatus().getConditions()
.stream().filter(c -> "Available".equals(c.getType())).findAny()
@ -66,70 +97,250 @@ class BasicTests {
fail("vm-operator not deployed.");
}
@AfterAll
static void tearDownAfterClass() throws Exception {
// Bring down manager
mgrDeployment.scale(0);
}
@Test
void test() throws IOException, InterruptedException, ApiException {
// Load from Yaml
var rdr = new FileReader("test-resources/unittest-vm.yaml");
var vmStub = K8sDynamicStub.createFromYaml(client, vmsContext, rdr);
assertTrue(vmStub.model().isPresent());
// Wait for created resources
assertTrue(waitForConfigMap(client));
assertTrue(waitForPvc(client));
// Check config map
var config = K8sV1ConfigMapStub.get(client, "vmop-dev", "unittest-vm")
.model().get();
var yaml = new Yaml(new SafeConstructor(new LoaderOptions()))
.load(config.getData().get("config.yaml"));
@SuppressWarnings("unchecked")
var maximumRam = ((Map<String, Map<String, Map<String, String>>>) yaml)
.get("/Runner").get("vm").get("maximumRam");
assertEquals("4 GiB", maximumRam);
// Cleanup
K8sDynamicStub.get(client, vmsContext, "vmop-dev", "unittest-vm")
.delete();
private static void deletePvcs() throws ApiException {
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector(
"app.kubernetes.io/managed-by=" + VM_OP_NAME + ","
+ "app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=unittest-vm");
+ "app.kubernetes.io/instance=" + VM_NAME);
var knownPvcs = K8sV1PvcStub.list(client, "vmop-dev", listOpts);
for (var pvc : knownPvcs) {
pvc.delete();
}
}
private boolean waitForConfigMap(K8sClient client)
throws InterruptedException, ApiException {
var stub = K8sV1ConfigMapStub.get(client, "vmop-dev", "unittest-vm");
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
return true;
}
Thread.sleep(1000);
}
return false;
@AfterAll
static void tearDownAfterClass() throws Exception {
// Cleanup
K8sDynamicStub.get(client, vmsContext, "vmop-dev", VM_NAME)
.delete();
deletePvcs();
// Bring down manager
mgrDeployment.scale(0);
}
private boolean waitForPvc(K8sClient client)
throws InterruptedException, ApiException {
var stub
= K8sV1PvcStub.get(client, "vmop-dev", "unittest-vm-runner-data");
@Test
void testConfigMap()
throws IOException, InterruptedException, ApiException {
K8sV1ConfigMapStub stub
= K8sV1ConfigMapStub.get(client, "vmop-dev", VM_NAME);
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
return true;
break;
}
Thread.sleep(1000);
}
return false;
// Check config map
var config = stub.model().get();
Map<List<? extends Object>, Object> toCheck = Map.of(
List.of("namespace"), "vmop-dev",
List.of("name"), VM_NAME,
List.of("labels", "app.kubernetes.io/name"), Constants.APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/managed-by"),
Constants.VM_OP_NAME,
List.of("annotations", "vmoperator.jdrupes.org/version"), EXISTS,
List.of("ownerReferences", 0, "apiVersion"),
vmsContext.getGroup() + "/" + vmsContext.getVersions().get(0),
List.of("ownerReferences", 0, "kind"), Constants.VM_OP_KIND_VM,
List.of("ownerReferences", 0, "name"), VM_NAME,
List.of("ownerReferences", 0, "uid"), EXISTS);
checkProps(config.getMetadata(), toCheck);
toCheck = new LinkedHashMap<>();
toCheck.put(List.of("/Runner", "guestShutdownStops"), false);
toCheck.put(List.of("/Runner", "cloudInit", "metaData", "instance-id"),
EXISTS);
toCheck.put(
List.of("/Runner", "cloudInit", "metaData", "local-hostname"),
VM_NAME);
toCheck.put(List.of("/Runner", "cloudInit", "userData"), Map.of());
toCheck.put(List.of("/Runner", "vm", "maximumRam"), "4 GiB");
toCheck.put(List.of("/Runner", "vm", "currentRam"), "2 GiB");
toCheck.put(List.of("/Runner", "vm", "maximumCpus"), 4);
toCheck.put(List.of("/Runner", "vm", "currentCpus"), 2);
toCheck.put(List.of("/Runner", "vm", "powerdownTimeout"), 1);
toCheck.put(List.of("/Runner", "vm", "network", 0, "type"), "user");
toCheck.put(List.of("/Runner", "vm", "drives", 0, "type"), "ide-cd");
toCheck.put(List.of("/Runner", "vm", "drives", 0, "file"),
"https://test.com/test.iso");
toCheck.put(List.of("/Runner", "vm", "drives", 0, "bootindex"), 0);
toCheck.put(List.of("/Runner", "vm", "drives", 1, "type"), "ide-cd");
toCheck.put(List.of("/Runner", "vm", "drives", 1, "file"),
"/var/local/vmop-image-repository/image.iso");
toCheck.put(List.of("/Runner", "vm", "drives", 2, "type"), "raw");
toCheck.put(List.of("/Runner", "vm", "drives", 2, "resource"),
"/dev/system-disk");
toCheck.put(List.of("/Runner", "vm", "drives", 3, "type"), "raw");
toCheck.put(List.of("/Runner", "vm", "drives", 3, "resource"),
"/dev/disk-1");
toCheck.put(List.of("/Runner", "vm", "display", "outputs"), 2);
toCheck.put(List.of("/Runner", "vm", "display", "spice", "port"), 5812);
toCheck.put(
List.of("/Runner", "vm", "display", "spice", "usbRedirects"), 2);
var cm = new Yaml(new SafeConstructor(new LoaderOptions()))
.load(config.getData().get("config.yaml"));
checkProps(cm, toCheck);
}
@Test
void testDisplaySecret() throws ApiException, InterruptedException {
ListOptions listOpts = new ListOptions();
listOpts.setLabelSelector("app.kubernetes.io/name=" + APP_NAME + ","
+ "app.kubernetes.io/instance=" + VM_NAME + ","
+ "app.kubernetes.io/component=" + COMP_DISPLAY_SECRET);
Collection<K8sV1SecretStub> secrets = null;
for (int i = 0; i < 10; i++) {
secrets = K8sV1SecretStub.list(client, "vmop-dev", listOpts);
if (secrets.size() > 0) {
break;
}
Thread.sleep(1000);
}
assertEquals(1, secrets.size());
var secretData = secrets.iterator().next().model().get().getData();
checkProps(secretData, Map.of(
List.of("display-password"), EXISTS));
assertEquals("now", new String(secretData.get("password-expiry")));
}
@Test
void testRunnerPvc() throws ApiException, InterruptedException {
var stub
= K8sV1PvcStub.get(client, "vmop-dev", VM_NAME + "-runner-data");
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
break;
}
Thread.sleep(1000);
}
var pvc = stub.model().get();
checkProps(pvc.getMetadata(), Map.of(
List.of("labels", "app.kubernetes.io/name"), Constants.APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/managed-by"),
Constants.VM_OP_NAME));
checkProps(pvc.getSpec(), Map.of(
List.of("resources", "requests", "storage"),
Quantity.fromString("1Mi")));
}
@Test
void testSystemDiskPvc() throws ApiException, InterruptedException {
var stub
= K8sV1PvcStub.get(client, "vmop-dev", VM_NAME + "-system-disk");
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
break;
}
Thread.sleep(1000);
}
var pvc = stub.model().get();
checkProps(pvc.getMetadata(), Map.of(
List.of("labels", "app.kubernetes.io/name"), Constants.APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/managed-by"),
Constants.VM_OP_NAME,
List.of("annotations", "use_as"), "system-disk"));
checkProps(pvc.getSpec(), Map.of(
List.of("resources", "requests", "storage"),
Quantity.fromString("1Gi")));
}
@Test
void testDisk1Pvc() throws ApiException, InterruptedException {
var stub
= K8sV1PvcStub.get(client, "vmop-dev", VM_NAME + "-disk-1");
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
break;
}
Thread.sleep(1000);
}
var pvc = stub.model().get();
checkProps(pvc.getMetadata(), Map.of(
List.of("labels", "app.kubernetes.io/name"), Constants.APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/managed-by"),
Constants.VM_OP_NAME));
checkProps(pvc.getSpec(), Map.of(
List.of("resources", "requests", "storage"),
Quantity.fromString("1Gi")));
}
@Test
void testPod() throws ApiException, InterruptedException {
PatchOptions opts = new PatchOptions();
opts.setForce(true);
opts.setFieldManager("kubernetes-java-kubectl-apply");
assertTrue(vmStub.patch(V1Patch.PATCH_FORMAT_JSON_PATCH,
new V1Patch("[{\"op\": \"replace\", \"path\": \"/spec/vm/state"
+ "\", \"value\": \"Running\"}]"),
client.defaultPatchOptions()).isPresent());
var stub = K8sV1PodStub.get(client, "vmop-dev", VM_NAME);
for (int i = 0; i < 20; i++) {
if (stub.model().isPresent()) {
break;
}
Thread.sleep(1000);
}
var pod = stub.model().get();
checkProps(pod.getMetadata(), Map.of(
List.of("labels", "app.kubernetes.io/name"), APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/component"), APP_NAME,
List.of("labels", "app.kubernetes.io/managed-by"),
Constants.VM_OP_NAME,
List.of("annotations", "vmrunner.jdrupes.org/cmVersion"), EXISTS,
List.of("annotations", "vmoperator.jdrupes.org/version"), EXISTS,
List.of("ownerReferences", 0, "apiVersion"),
vmsContext.getGroup() + "/" + vmsContext.getVersions().get(0),
List.of("ownerReferences", 0, "kind"), Constants.VM_OP_KIND_VM,
List.of("ownerReferences", 0, "name"), VM_NAME,
List.of("ownerReferences", 0, "uid"), EXISTS));
checkProps(pod.getSpec(), Map.of(
List.of("containers", 0, "image"), EXISTS,
List.of("containers", 0, "name"), VM_NAME,
List.of("containers", 0, "resources", "requests", "cpu"),
Quantity.fromString("1")));
}
@Test
public void testLoadBalancer() throws ApiException, InterruptedException {
var stub = K8sV1ServiceStub.get(client, "vmop-dev", VM_NAME);
for (int i = 0; i < 10; i++) {
if (stub.model().isPresent()) {
break;
}
Thread.sleep(1000);
}
var svc = stub.model().get();
checkProps(svc.getMetadata(), Map.of(
List.of("labels", "app.kubernetes.io/name"), APP_NAME,
List.of("labels", "app.kubernetes.io/instance"), VM_NAME,
List.of("labels", "app.kubernetes.io/managed-by"), VM_OP_NAME,
List.of("labels", "label1"), "label1",
List.of("labels", "label2"), "replaced",
List.of("labels", "label3"), "added",
List.of("annotations", "metallb.universe.tf/loadBalancerIPs"),
"192.168.168.1",
List.of("annotations", "anno1"), "added"));
}
private void checkProps(Object obj,
Map<? extends List<? extends Object>, Object> toCheck) {
for (var entry : toCheck.entrySet()) {
var prop = DataPath.get(obj, entry.getKey().toArray());
assertTrue(prop.isPresent(), () -> "Property " + entry.getKey()
+ " not found in " + obj);
// Check for existance only
if (entry.getValue() == EXISTS) {
continue;
}
assertEquals(entry.getValue(), prop.get());
}
}
}