mnl/VM-Operator
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 4 Wiki Activity Actions

Working creation and removal.

Browse source
This commit is contained in:
Michael Lipp 2023-07-28 15:43:50 +02:00
parent 954780e40f
commit dc1962af79
5 changed files with 316 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

1
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml
View file

@ -144,6 +144,7 @@ data:
</#if>
- type: raw
resource: /dev/disk-${ name }
<#assign drvCounter = drvCounter + 1/>
</#list>
display:

50
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerPod.ftl.yaml
View file

@ -23,16 +23,23 @@ spec:
# name: dev-tun
# - mountPath: /sys/fs/cgroup
# name: cgroup
- mountPath: /etc/opt/vmrunner
name: config
- mountPath: /var/local/vm-data
name: vm-data
<#--
- name: config
mountPath: /etc/opt/vmrunner
- name: vm-data
mountPath: /var/local/vm-data
volumeDevices:
{{- range $index, $disk := .Values.vm.disks }}
- devicePath: /dev/disk-{{ $index }}
name: disk-{{ $index }}
{{- end }}
<#assign diskCounter = 0/>
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.name??>
<#assign name = disk.volumeClaimTemplate.metadata.name.asString>
<#else>
<#assign name = "" + diskCounter>
</#if>
- name: disk-${ name }
devicePath: /dev/disk-${ name }
<#assign diskCounter = diskCounter + 1/>
</#list>
securityContext:
privileged: true
volumes:
@ -50,16 +57,23 @@ spec:
# path: /sys/fs/cgroup
- name: config
configMap:
name: {{ $.Release.Name }}
name: ${ cr.metadata.name.asString }
- name: vm-data
hostPath:
path: /var/local/vmrunner/{{ .Release.Name }}
{{- range $index, $disk := .Values.vm.disks }}
- name: disk-{{ $index }}
persistentVolumeClaim:
claimName: {{ $.Release.Name }}-pvc-{{ $index }}
{{- end }}
claimName: ${ cr.metadata.name.asString }-data
<#assign diskCounter = 0/>
<#list cr.spec.vm.disks.asList() as disk>
<#if disk.volumeClaimTemplate.metadata??
&& disk.volumeClaimTemplate.metadata.name??>
<#assign name = disk.volumeClaimTemplate.metadata.name.asString>
<#else>
<#assign name = "" + diskCounter>
</#if>
- name: disk-${ name }
persistentVolumeClaim:
claimName: ${ cr.metadata.name.asString }-disk-${ name }
<#assign diskCounter = diskCounter + 1/>
</#list>
hostNetwork: true
terminationGracePeriodSeconds: 60
terminationGracePeriodSeconds: ${ (cr.spec.vm.powerdownTimeout.asInt + 5)?c }
restartPolicy: Never
-->

16
org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/vmDataPvc.ftl.yaml Normal file
View file

@ -0,0 +1,16 @@
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
namespace: ${ cr.metadata.namespace.asString }
name: ${ cr.metadata.name.asString + "-data" }
labels:
app.kubernetes.io/name: ${ constants.APP_NAME }
app.kubernetes.io/instance: ${ cr.metadata.name.asString }
app.kubernetes.io/managed-by: ${ constants.VM_OP_NAME }
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Mi

136
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/K8s.java Normal file
View file

@ -0,0 +1,136 @@
/*
* VM-Operator
* Copyright (C) 2023 Michael N. Lipp
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.jdrupes.vmoperator.manager;
import io.kubernetes.client.common.KubernetesListObject;
import io.kubernetes.client.common.KubernetesObject;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiClient;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.openapi.models.V1ConfigMap;
import io.kubernetes.client.openapi.models.V1ConfigMapList;
import io.kubernetes.client.openapi.models.V1ObjectMeta;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaim;
import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimList;
import io.kubernetes.client.openapi.models.V1Pod;
import io.kubernetes.client.openapi.models.V1PodList;
import io.kubernetes.client.util.generic.GenericKubernetesApi;
import io.kubernetes.client.util.generic.options.PatchOptions;
import java.util.Optional;
/**
* Helpers for K8s API.
*/
@SuppressWarnings({ "PMD.ShortClassName", "PMD.UseUtilityClass" })
public class K8s {
/**
* Get PVC API.
*
* @param client the client
* @return the generic kubernetes api
*/
public static GenericKubernetesApi<V1PersistentVolumeClaim,
V1PersistentVolumeClaimList> pvcApi(ApiClient client) {
return new GenericKubernetesApi<>(V1PersistentVolumeClaim.class,
V1PersistentVolumeClaimList.class, "", "v1",
"persistentvolumeclaims", client);
}
/**
* Get config map API.
*
* @param client the client
* @return the generic kubernetes api
*/
public static GenericKubernetesApi<V1ConfigMap,
V1ConfigMapList> cmApi(ApiClient client) {
return new GenericKubernetesApi<>(V1ConfigMap.class,
V1ConfigMapList.class, "", "v1", "configmaps", client);
}
/**
* Get pod API.
*
* @param client the client
* @return the generic kubernetes api
*/
public static GenericKubernetesApi<V1Pod, V1PodList>
podApi(ApiClient client) {
return new GenericKubernetesApi<>(V1Pod.class, V1PodList.class, "",
"v1", "pods", client);
}
/**
* Get an object from its metadata.
*
* @param <T> the generic type
* @param <LT> the generic type
* @param api the api
* @param meta the meta
* @return the object
*/
public static <T extends KubernetesObject, LT extends KubernetesListObject>
Optional<T> get(GenericKubernetesApi<T, LT> api, V1ObjectMeta meta)
throws ApiException {
var response = api.get(meta.getNamespace(), meta.getName());
if (response.isSuccess()) {
return Optional.of(response.getObject());
}
return Optional.empty();
}
/**
* Delete an object.
*
* @param <T> the generic type
* @param <LT> the generic type
* @param api the api
* @param object the object
*/
public static <T extends KubernetesObject, LT extends KubernetesListObject>
void delete(GenericKubernetesApi<T, LT> api, T object)
throws ApiException {
api.delete(object.getMetadata().getNamespace(),
object.getMetadata().getName()).throwsApiException();
}
/**
* Apply the given patch data.
*
* @param <T> the generic type
* @param <LT> the generic type
* @param api the api
* @param existing the existing
* @param update the update
* @throws ApiException the api exception
*/
public static <T extends KubernetesObject, LT extends KubernetesListObject>
void
apply(GenericKubernetesApi<T, LT> api, T existing, String update)
throws ApiException {
PatchOptions opts = new PatchOptions();
opts.setForce(false);
opts.setFieldManager("kubernetes-java-kubectl-apply");
api.patch(existing.getMetadata().getNamespace(),
existing.getMetadata().getName(), V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(update), opts).throwsApiException();
}
}

186
org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Reconciler.java
View file

@ -29,11 +29,11 @@ import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import freemarker.template.TemplateHashModel;
import freemarker.template.TemplateNotFoundException;
import io.kubernetes.client.custom.V1Patch;
import io.kubernetes.client.openapi.ApiException;
import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesApi;
import io.kubernetes.client.util.generic.dynamic.DynamicKubernetesObject;
import io.kubernetes.client.util.generic.options.PatchOptions;
import io.kubernetes.client.util.generic.dynamic.Dynamics;
import io.kubernetes.client.util.generic.options.ListOptions;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Collections;
@ -43,6 +43,7 @@ import static org.jdrupes.vmoperator.manager.Constants.APP_NAME;
import static org.jdrupes.vmoperator.manager.Constants.VM_OP_GROUP;
import static org.jdrupes.vmoperator.manager.Constants.VM_OP_NAME;
import static org.jdrupes.vmoperator.manager.Constants.VM_OP_VERSION;
import org.jdrupes.vmoperator.manager.VmDefChanged.Type;
import org.jdrupes.vmoperator.util.ExtendedObjectWrapper;
import org.jgrapes.core.Channel;
import org.jgrapes.core.Component;
@ -90,29 +91,70 @@ public class Reconciler extends Component {
* @throws KubectlException
*/
@Handler
@SuppressWarnings("PMD.ConfusingTernary")
public void onVmDefChanged(VmDefChanged event, WatchChannel channel)
throws ApiException, TemplateNotFoundException,
MalformedTemplateNameException, ParseException, IOException,
TemplateException {
throws ApiException, TemplateException, IOException {
DynamicKubernetesApi vmDefApi = new DynamicKubernetesApi(VM_OP_GROUP,
VM_OP_VERSION, event.crd().getName(), channel.client());
var defMeta = event.metadata();
var vmDef = vmDefApi.get(defMeta.getNamespace(), defMeta.getName())
.getObject();
// Get common data for all reconciles
DynamicKubernetesObject vmDef = null;
Map<String, Object> model = null;
if (event.type() != Type.DELETED) {
vmDef = K8s.get(vmDefApi, defMeta).get();
// Prepare Freemarker model
@SuppressWarnings("PMD.UseConcurrentHashMap")
Map<String, Object> model = new HashMap<>();
model = new HashMap<>();
model.put("cr", vmDef.getRaw());
model.put("constants",
(TemplateHashModel) new DefaultObjectWrapperBuilder(
Configuration.VERSION_2_3_32)
.build().getStaticModels().get(Constants.class.getName()));
.build().getStaticModels()
.get(Constants.class.getName()));
}
// Reconcile
if (event.type() != Type.DELETED) {
reconcileDataPvc(model, channel);
reconcileDisks(vmDef, channel);
reconcileConfigMap(model, channel);
reconcilePod(model, channel);
reconcileConfigMap(event, model, channel);
reconcilePod(event, model, channel);
} else {
reconcilePod(event, model, channel);
reconcileConfigMap(event, model, channel);
deletePvcs(event, channel);
}
}
@SuppressWarnings("PMD.ConfusingTernary")
private void reconcileDataPvc(Map<String, Object> model,
WatchChannel channel)
throws TemplateException, ApiException, IOException {
// Combine template and data and parse result
var fmTemplate = fmConfig.getTemplate("vmDataPvc.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var pvcDef = Dynamics.newFromYaml(out.toString());
// Get API and check if PVC exists
DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
"persistentvolumeclaims", channel.client());
var existing = K8s.get(pvcApi, pvcDef.getMetadata());
// If PVC does not exist, create. Else patch (apply)
if (existing.isEmpty()) {
pvcApi.create(pvcDef);
} else {
// spec is immutable, so mix in existing spec
GsonPtr.to(pvcDef.getRaw()).set("spec", GsonPtr
.to(existing.get().getRaw()).get(JsonObject.class, "spec")
.get().deepCopy());
K8s.apply(pvcApi, existing.get(),
channel.client().getJSON().serialize(pvcDef));
}
}
private void reconcileDisks(DynamicKubernetesObject vmDef,
@ -131,22 +173,22 @@ public class Reconciler extends Component {
int index, JsonObject diskDef, WatchChannel channel)
throws ApiException {
var pvcObject = new DynamicKubernetesObject();
var pvcDef = GsonPtr.to(pvcObject.getRaw());
var vmDef = GsonPtr.to(vmDefinition.getRaw());
var pvcRaw = GsonPtr.to(pvcObject.getRaw());
var vmRaw = GsonPtr.to(vmDefinition.getRaw());
var pvcTpl = GsonPtr.to(diskDef).to("volumeClaimTemplate");
// Copy base and metadata from template and add missing/additional data.
pvcObject.setApiVersion(pvcTpl.getAsString("apiVersion").get());
pvcObject.setKind(pvcTpl.getAsString("kind").get());
var vmName = vmDef.getAsString("metadata", "name").orElse("default");
pvcDef.get(JsonObject.class).add("metadata",
var vmName = vmRaw.getAsString("metadata", "name").orElse("default");
pvcRaw.get(JsonObject.class).add("metadata",
pvcTpl.to("metadata").get(JsonObject.class).deepCopy());
var defMeta = pvcDef.to("metadata");
var defMeta = pvcRaw.to("metadata");
defMeta.computeIfAbsent("namespace", () -> new JsonPrimitive(
vmDef.getAsString("metadata", "namespace").orElse("default")));
vmRaw.getAsString("metadata", "namespace").orElse("default")));
defMeta.computeIfAbsent("name", () -> new JsonPrimitive(
vmName + "-disk-" + index));
var pvcLbls = pvcDef.to("metadata", "labels");
var pvcLbls = pvcRaw.to("metadata", "labels");
pvcLbls.set("app.kubernetes.io/name", APP_NAME);
pvcLbls.set("app.kubernetes.io/instance", vmName);
pvcLbls.set("app.kubernetes.io/component", "disk");
@ -155,72 +197,94 @@ public class Reconciler extends Component {
// Get API and check if PVC exists
DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
"persistentvolumeclaims", channel.client());
var existing = pvcApi.get(defMeta.getAsString("namespace").get(),
defMeta.getAsString("name").get());
var existing = K8s.get(pvcApi, pvcObject.getMetadata());
// If PVC does not exist, create. Else patch (apply)
if (!existing.isSuccess()) {
if (existing.isEmpty()) {
// PVC does not exist yet, copy spec from template
pvcDef.get(JsonObject.class).add("spec",
pvcRaw.get(JsonObject.class).add("spec",
pvcTpl.to("spec").get(JsonObject.class).deepCopy());
pvcApi.create(pvcObject);
} else {
// spec is immutable, so mix in existing spec
pvcDef.set("spec", GsonPtr.to(existing.getObject().getRaw())
pvcRaw.set("spec", GsonPtr.to(existing.get().getRaw())
.to("spec").get().deepCopy());
PatchOptions opts = new PatchOptions();
opts.setForce(false);
opts.setFieldManager("kubernetes-java-kubectl-apply");
pvcApi.patch(pvcObject.getMetadata().getNamespace(),
pvcObject.getMetadata().getName(),
V1Patch.PATCH_FORMAT_APPLY_YAML,
new V1Patch(channel.client().getJSON().serialize(pvcObject)),
opts).throwsApiException();
K8s.apply(pvcApi, existing.get(),
channel.client().getJSON().serialize(pvcObject));
}
}
private void reconcileConfigMap(Map<String, Object> model,
WatchChannel channel)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
private void deletePvcs(VmDefChanged event, WatchChannel channel)
throws ApiException {
// Get API and check and list related
var pvcApi = K8s.pvcApi(channel.client());
var pvcs = pvcApi.list(event.metadata().getNamespace(),
new ListOptions().labelSelector(
"app.kubernetes.io/managed-by=" + VM_OP_NAME
+ ",app.kubernetes.io/name=" + APP_NAME
+ ",app.kubernetes.io/instance="
+ event.metadata().getName()));
for (var pvc : pvcs.getObject().getItems()) {
K8s.delete(pvcApi, pvc);
}
}
private void reconcileConfigMap(VmDefChanged event,
Map<String, Object> model, WatchChannel channel)
throws IOException, TemplateException, ApiException {
// Get API and check if exists
DynamicKubernetesApi cmApi = new DynamicKubernetesApi("", "v1",
"configmaps", channel.client());
var existing = K8s.get(cmApi, event.metadata());
// If deleted, delete
if (event.type() == Type.DELETED) {
if (existing.isPresent()) {
K8s.delete(cmApi, existing.get());
}
return;
}
// Combine template and data and parse result
var fmTemplate = fmConfig.getTemplate("runnerConfig.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var mapDef = Dynamics.newFromYaml(out.toString());
// Apply
PatchOptions opts = new PatchOptions();
opts.setForce(false);
opts.setFieldManager("kubernetes-java-kubectl-apply");
DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
"configmaps", channel.client());
var vmDef = GsonPtr.to((JsonObject) model.get("cr"));
pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(),
vmDef.getAsString("metadata", "name").get(),
V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()),
opts).throwsApiException();
K8s.apply(cmApi, mapDef, out.toString());
}
private void reconcilePod(VmDefChanged event, Map<String, Object> model,
WatchChannel channel)
throws IOException, TemplateException, ApiException {
// Check if exists
DynamicKubernetesApi podApi = new DynamicKubernetesApi("", "v1",
"pods", channel.client());
var existing = K8s.get(podApi, event.metadata());
// If deleted, delete
if (event.type() == Type.DELETED) {
if (existing.isPresent()) {
K8s.delete(podApi, existing.get());
}
return;
}
private void reconcilePod(Map<String, Object> model, WatchChannel channel)
throws TemplateNotFoundException, MalformedTemplateNameException,
ParseException, IOException, TemplateException, ApiException {
// Combine template and data and parse result
var fmTemplate = fmConfig.getTemplate("runnerPod.ftl.yaml");
StringWriter out = new StringWriter();
fmTemplate.process(model, out);
out = null;
// Avoid Yaml.load due to
// https://github.com/kubernetes-client/java/issues/2741
var podDef = Dynamics.newFromYaml(out.toString());
// // Apply
// PatchOptions opts = new PatchOptions();
// opts.setForce(false);
// opts.setFieldManager("kubernetes-java-kubectl-apply");
// DynamicKubernetesApi pvcApi = new DynamicKubernetesApi("", "v1",
// "pod", channel.client());
// var vmDef = GsonPtr.to((JsonObject) model.get("cr"));
// pvcApi.patch(vmDef.getAsString("metadata", "namespace").get(),
// vmDef.getAsString("metadata", "name").get(),
// V1Patch.PATCH_FORMAT_APPLY_YAML, new V1Patch(out.toString()),
// opts).throwsApiException();
// Nothing can be updated here
if (existing.isEmpty()) {
podApi.create(podDef);
}
}
}