diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7dff899..b7b04f0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ stages: - "*/build" before_script: - echo -n $CI_REGISTRY_PASSWORD | podman login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY - - git switch $(git branch -r --sort="authordate" --contains $CI_COMMIT_SHA | head -1 | sed -e 's#[^/]*/##') + - git switch $(git branch -r --sort="authordate" --contains $CI_COMMIT_SHA | head -1 | sed -e 's#.*/##') - git pull - git reset --hard $CI_COMMIT_SHA diff --git a/deploy/vmop-deployment.yaml b/deploy/vmop-deployment.yaml index 61dd21b..dcbf133 100644 --- a/deploy/vmop-deployment.yaml +++ b/deploy/vmop-deployment.yaml @@ -20,7 +20,7 @@ spec: containers: - name: vm-operator image: >- - ghcr.io/mnlipp/org.jdrupes.vmoperator.manager:4.0.1 + ghcr.io/mnlipp/org.jdrupes.vmoperator.manager:4.0.2 env: - name: JAVA_OPTS # The VM operator needs about 25 MB of memory, plus 1 MB for diff --git a/org.jdrupes.vmoperator.manager/build.gradle b/org.jdrupes.vmoperator.manager/build.gradle index eda5ce0..4ce4ed0 100644 --- a/org.jdrupes.vmoperator.manager/build.gradle +++ b/org.jdrupes.vmoperator.manager/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation 'org.jgrapes:org.jgrapes.io:[2.12.1,3)' implementation 'org.jgrapes:org.jgrapes.http:[3.5.0,4)' - implementation 'org.jgrapes:org.jgrapes.webconsole.base:[2.2.0,3)' + implementation 'org.jgrapes:org.jgrapes.webconsole.base:[2.3.0,3)' implementation 'org.jgrapes:org.jgrapes.webconsole.vuejs:[1.8.0,2)' implementation 'org.jgrapes:org.jgrapes.webconsole.rbac:[1.4.0,2)' implementation 'org.jgrapes:org.jgrapes.webconlet.oidclogin:[1.7.0,2)' diff --git a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml index 5dc8d9b..0200021 100644 --- a/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml +++ b/org.jdrupes.vmoperator.manager/resources/org/jdrupes/vmoperator/manager/runnerConfig.ftl.yaml @@ -37,7 +37,7 @@ data: # The template to use. Resolved relative to /usr/share/vmrunner/templates. # template: "Standard-VM-latest.ftl.yaml" <#if spec.runnerTemplate?? && spec.runnerTemplate.source?? > - template: ${ cm.spec().runnerTemplate.source } + template: ${ spec.runnerTemplate.source } # The template is copied to the data diretory when the VM starts for diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java index c15acc5..ce14488 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/Controller.java @@ -50,6 +50,7 @@ import org.jdrupes.vmoperator.manager.events.VmPoolChanged; import org.jdrupes.vmoperator.manager.events.VmResourceChanged; import org.jgrapes.core.Channel; import org.jgrapes.core.Component; +import org.jgrapes.core.EventPipeline; import org.jgrapes.core.annotation.Handler; import org.jgrapes.core.events.HandlingError; import org.jgrapes.core.events.Start; @@ -94,7 +95,7 @@ import org.jgrapes.util.events.ConfigurationUpdate; public class Controller extends Component { private String namespace; - private final ChannelManager chanMgr; + private final ChannelManager chanMgr; /** * Creates a new instance. diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java index 60d27d4..1e3eb0f 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/DisplaySecretReconciler.java @@ -66,7 +66,6 @@ import org.jose4j.base64url.Base64; * * `passwordValidity`: the validity of the random password in seconds. * Used to calculate the password expiry time in the generated secret. */ -@SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", "PMD.TooManyStaticImports" }) public class DisplaySecretReconciler extends Component { protected final Logger logger = Logger.getLogger(getClass().getName()); @@ -104,12 +103,15 @@ public class DisplaySecretReconciler extends Component { return oldConfig; }).ifPresent(c -> { try { - if (c.containsKey("passwordValidity")) { - passwordValidity = Integer - .parseInt((String) c.get("passwordValidity")); - } - } catch (ClassCastException e) { - logger.config("Malformed configuration: " + e.getMessage()); + Optional.ofNullable(c.get("passwordValidity")) + .map(p -> p instanceof Integer ? (Integer) p + : Integer.valueOf((String) p)) + .ifPresent(p -> { + passwordValidity = p; + }); + } catch (NumberFormatException e) { + logger.warning( + () -> "Malformed configuration: " + e.getMessage()); } }); } @@ -190,7 +192,6 @@ public class DisplaySecretReconciler extends Component { * @throws ApiException the api exception */ @Handler - @SuppressWarnings("PMD.StringInstantiation") public void onGetDisplaySecret(GetDisplaySecret event, VmChannel channel) throws ApiException { // Get VM definition and check if running @@ -319,7 +320,6 @@ public class DisplaySecretReconciler extends Component { /** * The Class PendingGet. */ - @SuppressWarnings("PMD.DataClass") private static class PendingRequest { public final GetDisplaySecret event; public final long expectedSerial; diff --git a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java index 09bade8..b667aa6 100644 --- a/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java +++ b/org.jdrupes.vmoperator.manager/src/org/jdrupes/vmoperator/manager/VmMonitor.java @@ -22,7 +22,6 @@ import com.google.gson.JsonObject; import io.kubernetes.client.apimachinery.GroupVersionKind; import io.kubernetes.client.custom.V1Patch; import io.kubernetes.client.openapi.ApiException; -import io.kubernetes.client.openapi.models.V1ObjectMeta; import io.kubernetes.client.util.Watch; import io.kubernetes.client.util.generic.options.ListOptions; import java.io.IOException; @@ -32,7 +31,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Optional; import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.jdrupes.vmoperator.common.Constants.Crd; import org.jdrupes.vmoperator.common.Constants.Status; @@ -57,16 +55,28 @@ import org.jdrupes.vmoperator.manager.events.VmResourceChanged; import org.jdrupes.vmoperator.util.GsonPtr; import org.jgrapes.core.Channel; import org.jgrapes.core.Event; +import org.jgrapes.core.EventPipeline; import org.jgrapes.core.annotation.Handler; /** - * Watches for changes of VM definitions. + * Watches for changes of VM definitions. When a VM definition (CR) + * becomes known, is is registered with a {@link ChannelManager} and thus + * gets an associated {@link VmChannel} and an associated + * {@link EventPipeline}. + * + * The {@link EventPipeline} is used for submitting an action that processes + * the change data from kubernetes, eventually transforming it to a + * {@link VmResourceChanged} event that is handled by another + * {@link EventPipeline} associated with the {@link VmChannel}. This + * event pipeline should be used for all events related to changes of + * a particular VM. */ @SuppressWarnings({ "PMD.DataflowAnomalyAnalysis", "PMD.ExcessiveImports" }) public class VmMonitor extends AbstractMonitor { - private final ChannelManager channelManager; + private final ChannelManager channelManager; /** * Instantiates a new VM definition watcher. @@ -75,7 +85,7 @@ public class VmMonitor extends * @param channelManager the channel manager */ public VmMonitor(Channel componentChannel, - ChannelManager channelManager) { + ChannelManager channelManager) { super(componentChannel, VmDefinition.class, VmDefinitions.class); this.channelManager = channelManager; @@ -122,14 +132,18 @@ public class VmMonitor extends @Override protected void handleChange(K8sClient client, Watch.Response response) { - V1ObjectMeta metadata = response.object.getMetadata(); - AtomicBoolean toBeAdded = new AtomicBoolean(false); - VmChannel channel = channelManager.channel(metadata.getName()) - .orElseGet(() -> { - toBeAdded.set(true); - return channelManager.createChannel(metadata.getName()); - }); + var name = response.object.getMetadata().getName(); + // Process the response data on a VM specific pipeline to + // increase concurrency when e.g. starting many VMs. + var preparing = channelManager.associated(name) + .orElseGet(() -> newEventPipeline()); + preparing.submit("VmChange[" + name + "]", + () -> processChange(client, response, preparing)); + } + + private void processChange(K8sClient client, + Watch.Response response, EventPipeline preparing) { // Get full definition and associate with channel as backup var vmDef = response.object; if (vmDef.data() == null) { @@ -137,6 +151,9 @@ public class VmMonitor extends // https://github.com/kubernetes-client/java/issues/3215 vmDef = getModel(client, vmDef); } + var name = response.object.getMetadata().getName(); + var channel = channelManager.channel(name) + .orElseGet(() -> channelManager.createChannel(name)); if (vmDef.data() != null) { // New data, augment and save addExtraData(vmDef, channel.vmDefinition()); @@ -150,9 +167,7 @@ public class VmMonitor extends + response.object.getMetadata()); return; } - if (toBeAdded.get()) { - channelManager.put(vmDef.name(), channel); - } + channelManager.put(name, channel, preparing); // Create and fire changed event. Remove channel from channel // manager on completion. @@ -199,9 +214,16 @@ public class VmMonitor extends @Handler public void onPodChanged(PodChanged event, VmChannel channel) { var vmDef = channel.vmDefinition(); - updateNodeInfo(event, vmDef); - channel - .fire(new VmResourceChanged(ResponseType.MODIFIED, vmDef, false, true)); + + // Make sure that this is properly sync'd with VM CR changes. + channelManager.associated(vmDef.name()) + .orElseGet(() -> activeEventPipeline()) + .submit("NodeInfo[" + vmDef.name() + "]", + () -> { + updateNodeInfo(event, vmDef); + channel.fire(new VmResourceChanged(ResponseType.MODIFIED, + vmDef, false, true)); + }); } private void updateNodeInfo(PodChanged event, VmDefinition vmDef) { diff --git a/org.jdrupes.vmoperator.vmmgmt/resources/org/jdrupes/vmoperator/vmmgmt/VmMgmt-view.ftl.html b/org.jdrupes.vmoperator.vmmgmt/resources/org/jdrupes/vmoperator/vmmgmt/VmMgmt-view.ftl.html index 5a28cb8..3197440 100644 --- a/org.jdrupes.vmoperator.vmmgmt/resources/org/jdrupes/vmoperator/vmmgmt/VmMgmt-view.ftl.html +++ b/org.jdrupes.vmoperator.vmmgmt/resources/org/jdrupes/vmoperator/vmmgmt/VmMgmt-view.ftl.html @@ -30,7 +30,7 @@ -