From 137571199b3c6a92d5df5e36eb80df4f8240da8c Mon Sep 17 00:00:00 2001 From: Troy Mohl Date: Mon, 9 Mar 2020 15:22:24 -0400 Subject: [PATCH 1/5] Fix saving of EC2 connections using alternate URL configuration --- .../hudson/plugins/ec2/AmazonEC2Cloud.java | 24 ++++++++++++------- .../ec2/util/AmazonEC2FactoryImpl.java | 9 ++++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java index c04304f6e..4a3120e0a 100644 --- a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java @@ -24,6 +24,10 @@ package hudson.plugins.ec2; import com.amazonaws.SdkClientException; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.model.DescribeRegionsResult; +import com.amazonaws.services.ec2.model.Region; import com.google.common.annotations.VisibleForTesting; import hudson.Extension; import hudson.Util; @@ -32,7 +36,6 @@ import hudson.slaves.Cloud; import hudson.util.FormValidation; import hudson.util.ListBoxModel; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -40,18 +43,10 @@ import java.util.Locale; import javax.annotation.Nullable; import javax.servlet.ServletException; - import jenkins.model.Jenkins; - import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.StaplerResponse; - -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.ec2.AmazonEC2; -import com.amazonaws.services.ec2.model.DescribeRegionsResult; -import com.amazonaws.services.ec2.model.Region; import org.kohsuke.stapler.interceptor.RequirePOST; /** @@ -65,6 +60,8 @@ public class AmazonEC2Cloud extends EC2Cloud { */ private String region; + private String altEC2Endpoint; + public static final String CLOUD_ID_PREFIX = "ec2-"; private boolean noDelayProvisioning; @@ -129,6 +126,15 @@ public void setNoDelayProvisioning(boolean noDelayProvisioning) { this.noDelayProvisioning = noDelayProvisioning; } + public String getAltEC2Endpoint() { + return altEC2Endpoint; + } + + @DataBoundSetter + public void setAltEC2Endpoint(String altEC2Endpoint) { + this.altEC2Endpoint = altEC2Endpoint; + } + @Override protected AWSCredentialsProvider createCredentialsProvider() { return createCredentialsProvider(isUseInstanceProfileForCredentials(), getCredentialsId(), getRoleArn(), getRoleSessionName(), getRegion()); diff --git a/src/main/java/hudson/plugins/ec2/util/AmazonEC2FactoryImpl.java b/src/main/java/hudson/plugins/ec2/util/AmazonEC2FactoryImpl.java index e685c4fbb..289575ffc 100644 --- a/src/main/java/hudson/plugins/ec2/util/AmazonEC2FactoryImpl.java +++ b/src/main/java/hudson/plugins/ec2/util/AmazonEC2FactoryImpl.java @@ -14,8 +14,11 @@ public class AmazonEC2FactoryImpl implements AmazonEC2Factory { @Override public AmazonEC2 connect(AWSCredentialsProvider credentialsProvider, URL ec2Endpoint) { - AmazonEC2 client = new AmazonEC2Client(credentialsProvider, EC2Cloud.createClientConfiguration(ec2Endpoint.getHost())); - client.setEndpoint(ec2Endpoint.toString()); - return client; + if (ec2Endpoint != null && ec2Endpoint.toString().trim().length() > 0) { + AmazonEC2 client = new AmazonEC2Client(credentialsProvider, EC2Cloud.createClientConfiguration(ec2Endpoint.getHost())); + client.setEndpoint( ec2Endpoint.toString() ); + return client; + } + return null; } } From 99b4ec64985729e1b554574cb320ee153ed57343 Mon Sep 17 00:00:00 2001 From: Troy Mohl Date: Tue, 10 Mar 2020 06:54:02 -0400 Subject: [PATCH 2/5] Stop/Restart of existing DumbSlave nodes that are EC2 backed --- .../hudson/plugins/ec2/AmazonEC2Cloud.java | 199 ++++++++++++++++++ .../hudson/plugins/ec2/InstanceStopTimer.java | 128 +++++++++++ .../ec2/StartInstanceProvisionerStrategy.java | 125 +++++++++++ .../ec2/AmazonEC2Cloud/config-entries.jelly | 12 ++ .../help-instanceTagForJenkins.html | 1 + .../AmazonEC2Cloud/help-nodeTagForEc2.html | 3 + .../AmazonEC2Cloud/help-startStopNodes.html | 3 + .../plugins/ec2/InstanceStopTimerTest.java | 103 +++++++++ .../StartInstanceProvisionerStrategyTest.java | 108 ++++++++++ 9 files changed, 682 insertions(+) create mode 100644 src/main/java/hudson/plugins/ec2/InstanceStopTimer.java create mode 100644 src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java create mode 100644 src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-instanceTagForJenkins.html create mode 100644 src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html create mode 100644 src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-startStopNodes.html create mode 100644 src/test/java/hudson/plugins/ec2/InstanceStopTimerTest.java create mode 100644 src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java diff --git a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java index 4a3120e0a..34e4be2ca 100644 --- a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java @@ -26,27 +26,44 @@ import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.model.DescribeInstancesRequest; +import com.amazonaws.services.ec2.model.DescribeInstancesResult; import com.amazonaws.services.ec2.model.DescribeRegionsResult; +import com.amazonaws.services.ec2.model.Filter; +import com.amazonaws.services.ec2.model.Instance; +import com.amazonaws.services.ec2.model.InstanceStateName; import com.amazonaws.services.ec2.model.Region; +import com.amazonaws.services.ec2.model.Reservation; +import com.amazonaws.services.ec2.model.StartInstancesRequest; +import com.amazonaws.services.ec2.model.StopInstancesRequest; import com.google.common.annotations.VisibleForTesting; import hudson.Extension; import hudson.Util; +import hudson.model.Computer; import hudson.model.Failure; +import hudson.model.Node; +import hudson.model.labels.LabelAtom; import hudson.plugins.ec2.util.AmazonEC2Factory; import hudson.slaves.Cloud; +import hudson.slaves.NodeProvisioner.PlannedNode; import hudson.util.FormValidation; import hudson.util.ListBoxModel; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nullable; import javax.servlet.ServletException; import jenkins.model.Jenkins; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; +import org.kohsuke.stapler.StaplerResponse; import org.kohsuke.stapler.interceptor.RequirePOST; /** @@ -62,10 +79,26 @@ public class AmazonEC2Cloud extends EC2Cloud { private String altEC2Endpoint; + private static final Logger LOGGER = Logger.getLogger(AmazonEC2Cloud.class.getName()); + public static final String CLOUD_ID_PREFIX = "ec2-"; + private static final int MAX_RESULTS = 1000; + + private static final String INSTANCE_NAME_TAG = "Name"; + + private static final String TAG_PREFIX = "tag"; + private boolean noDelayProvisioning; + private boolean startStopNodes; + + private String instanceTagForJenkins; + + private String nodeTagForEc2; + + private String maxIdleMinutes; + @DataBoundConstructor public AmazonEC2Cloud(String cloudName, boolean useInstanceProfileForCredentials, String credentialsId, String region, String privateKey, String instanceCapStr, List templates, String roleArn, String roleSessionName) { super(createCloudId(cloudName), useInstanceProfileForCredentials, credentialsId, privateKey, instanceCapStr, templates, roleArn, roleSessionName); @@ -126,6 +159,24 @@ public void setNoDelayProvisioning(boolean noDelayProvisioning) { this.noDelayProvisioning = noDelayProvisioning; } + @DataBoundSetter + public void setStartStopNodes(boolean startStopNodes) { + this.startStopNodes = startStopNodes; + } + + public boolean isStartStopNodes() { + return startStopNodes; + } + + public String getInstanceTagForJenkins() { + return instanceTagForJenkins; + } + + @DataBoundSetter + public void setInstanceTagForJenkins(String instanceTagForJenkins) { + this.instanceTagForJenkins = instanceTagForJenkins; + } + public String getAltEC2Endpoint() { return altEC2Endpoint; } @@ -135,11 +186,159 @@ public void setAltEC2Endpoint(String altEC2Endpoint) { this.altEC2Endpoint = altEC2Endpoint; } + public String getNodeTagForEc2() { + return nodeTagForEc2; + } + + @DataBoundSetter + public void setNodeTagForEc2(String nodeTagForEc2) { + this.nodeTagForEc2 = nodeTagForEc2; + } + + public boolean isEc2Node(Node node) { + //If no label is specified then we check all nodes + if (nodeTagForEc2 == null || nodeTagForEc2.trim().length() == 0) { + return true; + } + + for (LabelAtom label : node.getAssignedLabels()) { + if (label.getExpression().equalsIgnoreCase(nodeTagForEc2)) { + return true; + } + } + return false; + } + + public String getMaxIdleMinutes() { + return maxIdleMinutes; + } + + @DataBoundSetter + public void setMaxIdleMinutes(String maxIdleMinutes) { + this.maxIdleMinutes = maxIdleMinutes; + } + + public PlannedNode startNode(Node node) { + Instance nodeInstance = getInstanceByLabel(node.getSelfLabel().getExpression(), InstanceStateName.Stopped); + if (nodeInstance == null) { + nodeInstance = getInstanceByNodeName(node.getNodeName(), InstanceStateName.Stopped); + } + + if (nodeInstance == null) { + return null; + } + + final String instanceId = nodeInstance.getInstanceId(); + + return new PlannedNode(node.getDisplayName(), + Computer.threadPoolForRemoting.submit(() -> { + try { + while (true) { + StartInstancesRequest startRequest = new StartInstancesRequest(); + startRequest.setInstanceIds(Collections.singletonList(instanceId)); + connect().startInstances(startRequest); + + Instance instance = CloudHelper.getInstanceWithRetry(instanceId, this); + if (instance == null) { + LOGGER.log(Level.WARNING, "Can't find instance with instance id `{0}` in cloud {1}. Terminate provisioning ", new Object[] { + instanceId, this.getCloudName() }); + return null; + } + + InstanceStateName state = InstanceStateName.fromValue(instance.getState().getName()); + if (state.equals(InstanceStateName.Running)) { + long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - instance.getLaunchTime().getTime()); + LOGGER.log(Level.INFO, "{0} moved to RUNNING state in {1} seconds and is ready to be connected by Jenkins", new Object[] { + instanceId, startTime }); + return node; + } + + if (!state.equals(InstanceStateName.Pending)) { + LOGGER.log(Level.WARNING, "{0}. Node {1} is neither pending nor running, it's {2}. Terminate provisioning", new Object[] { + instanceId, node.getNodeName(), state }); + return null; + } + + Thread.sleep(5000); + } + } catch (Exception e) { + LOGGER.log(Level.WARNING, "Unable to start " + instanceId, e); + return null; + } + }) + , node.getNumExecutors()); + } + + public void stopNode(Node node) { + Instance nodeInstance = getInstanceByLabel(node.getSelfLabel().getExpression(), InstanceStateName.Running); + if (nodeInstance == null) { + nodeInstance = getInstanceByNodeName(node.getNodeName(), InstanceStateName.Running); + } + + if (nodeInstance == null) { + return; + } + + final String instanceId = nodeInstance.getInstanceId(); + + try { + StopInstancesRequest request = new StopInstancesRequest(); + request.setInstanceIds(Collections.singletonList(instanceId)); + connect().stopInstances(request); + LOGGER.log(Level.INFO, "Stopped instance: {0}", instanceId); + } catch (Exception e) { + LOGGER.log(Level.INFO, "Unable to stop instance: " + instanceId, e); + } + } + @Override protected AWSCredentialsProvider createCredentialsProvider() { return createCredentialsProvider(isUseInstanceProfileForCredentials(), getCredentialsId(), getRoleArn(), getRoleSessionName(), getRegion()); } + private Instance getInstanceByLabel(String label, InstanceStateName desiredState) { + String tag = getInstanceTagForJenkins(); + if (tag == null) { + return null; + } + return getInstance(Collections.singletonList(getTagFilter(tag, label)), desiredState); + } + + private Instance getInstanceByNodeName(String name, InstanceStateName desiredState) { + return getInstance(Collections.singletonList(getTagFilter(INSTANCE_NAME_TAG, name)), desiredState); + } + + private Filter getTagFilter(String name, String value) { + Filter filter = new Filter(); + filter.setName(TAG_PREFIX + ":" + name.trim()); + filter.setValues(Collections.singletonList(value.trim())); + LOGGER.log(Level.FINEST,"Created filter to query for instance: {0}", filter); + return filter; + } + + private Instance getInstance(List filters, InstanceStateName desiredState) { + DescribeInstancesRequest request = new DescribeInstancesRequest(); + request.setFilters(filters); + request.setMaxResults(MAX_RESULTS); + request.setNextToken(null); + DescribeInstancesResult response = connect().describeInstances( request ); + + if (!response.getReservations().isEmpty()) { + for (Reservation reservation : response.getReservations()) { + for (Instance instance : reservation.getInstances()) { + com.amazonaws.services.ec2.model.InstanceState state = instance.getState(); + LOGGER.log(Level.FINEST,"Instance {0} state: {1}", new Object[] {instance.getInstanceId(), state.getName()}); + if (state.getName().equals(desiredState.toString())) { + return instance; + } + } + } + } else { + LOGGER.log(Level.FINEST,"No instances found that matched filter criteria"); + } + return null; + } + @Extension public static class DescriptorImpl extends EC2Cloud.DescriptorImpl { diff --git a/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java b/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java new file mode 100644 index 000000000..96dd33ebe --- /dev/null +++ b/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java @@ -0,0 +1,128 @@ +package hudson.plugins.ec2; + +import com.google.common.annotations.VisibleForTesting; +import hudson.Extension; +import hudson.model.AsyncPeriodicWork; +import hudson.model.Computer; +import hudson.model.Executor; +import hudson.model.Node; +import hudson.model.TaskListener; +import hudson.slaves.Cloud; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import jenkins.model.Jenkins; + +@Extension +public class InstanceStopTimer extends AsyncPeriodicWork { + private static final Logger LOGGER = Logger.getLogger(InstanceStopTimer.class.getName()); + + private static final long STOP_DISABLED = -1; + + public InstanceStopTimer() { + super(InstanceStopTimer.class.getName()); + } + + protected InstanceStopTimer(String name) { + super(name); + } + + @Override protected void execute(TaskListener taskListener) throws IOException, InterruptedException { + Jenkins jenkinsInstance = Jenkins.get(); + for (Node node : jenkinsInstance.getNodes()) { + if (shouldStopNode(node)) { + LOGGER.log(Level.FINEST, "{0} should be stopped", node.getNodeName()); + stopNode(node); + } + } + } + + @Override public long getRecurrencePeriod() { + return TimeUnit.MINUTES.toMillis(1); + } + + private boolean shouldStopNode(Node node) { + long maxIdleMillis = getMaxIdleMillis(); + + if (maxIdleMillis < 0) { + return false; + } + boolean shouldStopNode = false; + Computer computer = getComputer(node); + if (computer != null && computer.isOnline() && !computer.isConnecting()) { + boolean executorWasUsed = false; + for (Executor executor : computer.getAllExecutors()) { + if (executor.isIdle()) { + long idleStart = executor.getIdleStartMilliseconds(); + long idleTime = System.currentTimeMillis() - idleStart; + LOGGER.log(Level.FINEST, "{0} executor: {1} has been idle for: {2}", new Object[] {node.getNodeName() ,executor.getDisplayName(), idleTime}); + if (idleTime < maxIdleMillis) { + executorWasUsed = true; + break; + } + } else { + executorWasUsed = true; + break; + } + } + shouldStopNode = !executorWasUsed; + } + return shouldStopNode; + } + + private void stopNode(Node node) { + Jenkins jenkinsInstance = Jenkins.get(); + + for (Cloud cloud : jenkinsInstance.clouds) { + if (!(cloud instanceof AmazonEC2Cloud)) + continue; + AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud; + if (ec2.isStartStopNodes() && ec2.isEc2Node(node)) { + LOGGER.log(Level.INFO, "Requesting stop on {0} of {1}", new Object[] {ec2.getCloudName(), node.getNodeName()}); + try { + ec2.stopNode(node); + } catch (Exception e) { + LOGGER.log(Level.INFO, "Unable to start an EC2 Instance for node: " + node.getNodeName(), e); + } + } + } + } + + private long getMaxIdleMillis() { + long maxMinutes = STOP_DISABLED; + Jenkins jenkinsInstance = Jenkins.get(); + for (Cloud cloud : jenkinsInstance.clouds) { + if (!(cloud instanceof AmazonEC2Cloud)) + continue; + AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud; + if (ec2.isStartStopNodes()) { + Integer configuredMax = getInteger(ec2.getMaxIdleMinutes()); + if (configuredMax != null) { + maxMinutes = Math.max(maxMinutes, configuredMax); + } + } + } + if (maxMinutes > 0) { + return TimeUnit.MINUTES.toMillis(maxMinutes); + } + return maxMinutes; + } + + private Integer getInteger(String str) { + if (str != null) { + try { + return Integer.parseInt(str); + } catch (NumberFormatException nfe) { + LOGGER.log(Level.INFO, "Couldn't get integer from string: {0}", str); + return null; + } + } + return null; + } + + @VisibleForTesting + protected Computer getComputer(Node node) { + return node.toComputer(); + } +} diff --git a/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java b/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java new file mode 100644 index 000000000..c1050df4b --- /dev/null +++ b/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java @@ -0,0 +1,125 @@ +package hudson.plugins.ec2; + +import com.google.common.annotations.VisibleForTesting; +import hudson.Extension; +import hudson.model.Computer; +import hudson.model.Label; +import hudson.model.LoadStatistics; +import hudson.model.Node; +import hudson.model.labels.LabelAtom; +import hudson.slaves.Cloud; +import hudson.slaves.NodeProvisioner; +import hudson.slaves.NodeProvisioner.PlannedNode; +import java.util.Collection; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.Logger; +import jenkins.model.Jenkins; + +/** + * Implementation of {@link NodeProvisioner.Strategy} which will attempt to restart EC2 nodes + * that are shut down to meet the demand. + * + */ +@Extension(ordinal = 101) +public class StartInstanceProvisionerStrategy extends NodeProvisioner.Strategy { + + private static final Logger LOGGER = Logger.getLogger(StartInstanceProvisionerStrategy.class.getName()); + + @Override + public NodeProvisioner.StrategyDecision apply(NodeProvisioner.StrategyState strategyState) { + final Label label = strategyState.getLabel(); + + LOGGER.log(Level.FINEST, "Calling into StartInstanceProvisionerStrategy for label: {0}", label.getExpression()); + + LoadStatistics.LoadStatisticsSnapshot snapshot = strategyState.getSnapshot(); + + int currentDemand = snapshot.getQueueLength(); + int availableCapacity = getCurrentCapacity(label) + + strategyState.getAdditionalPlannedCapacity() + + strategyState.getPlannedCapacitySnapshot(); + LOGGER.log(Level.FINE,"Demand: {0}, Avail Capacity: {1}", new Object[]{currentDemand, availableCapacity}); + + if (currentDemand > availableCapacity) { + Jenkins jenkinsInstance = Jenkins.get(); + LOGGER.log(Level.FINE, "Attempting to find node for label: {0}", label); + for (Node node : jenkinsInstance.getNodes()) { + if (nodeHasLabel(node, label.getExpression())) { + LOGGER.log(Level.FINE,"Found the node, checking if it's running"); + if (!isNodeOnline(node)) { + LOGGER.log(Level.FINE,"Attempting to start node: {0}", node.getNodeName()); + PlannedNode plannedNode = startNode(node); + if (plannedNode != null) { + Collection plannedNodes = Collections.singletonList(plannedNode); + LOGGER.log(Level.FINE, "Planned {0} new nodes", plannedNodes.size()); + strategyState.recordPendingLaunches(plannedNodes); + availableCapacity += plannedNodes.size(); + break; + } + } + } + } + } + if (availableCapacity >= currentDemand) { + LOGGER.log(Level.FINE, "Provisioning completed"); + return NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED; + } else { + LOGGER.log(Level.FINE, "Provisioning not complete, consulting remaining strategies"); + return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES; + } + } + + @VisibleForTesting + protected boolean isNodeOnline(Node node) { + Computer nodeComputer = node.toComputer(); + if (nodeComputer != null) { + return nodeComputer.isOnline(); + } + return false; + } + + private PlannedNode startNode(Node node) { + Jenkins jenkinsInstance = Jenkins.get(); + PlannedNode plannedNode = null; + + for (Cloud cloud : jenkinsInstance.clouds) { + if (!(cloud instanceof AmazonEC2Cloud)) + continue; + AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud; + if (ec2.isStartStopNodes() && ec2.isEc2Node(node)) { + LOGGER.log(Level.FINE, "Node on {0} of {1} not connected to Jenkins, should be started", new Object[] {ec2.getCloudName(), node.getNodeName()}); + try { + plannedNode = ec2.startNode(node); + } catch (Exception e) { + LOGGER.log(Level.INFO, "Unable to start an EC2 Instance for node: " + node.getNodeName(), e); + } + } + } + return plannedNode; + } + + private int getCurrentCapacity(Label label) { + int currentCapacity = 0; + Jenkins jenkinsInstance = Jenkins.get(); + for (Node node : jenkinsInstance.getNodes()) { + if (isNodeOnline(node)) { + Computer computer = node.toComputer(); + if (computer != null && computer.isOnline() && !computer.isConnecting()) { + if (nodeHasLabel(node, label.getExpression())) { + currentCapacity += node.getNumExecutors(); + } + } + } + } + return currentCapacity; + } + + private boolean nodeHasLabel(Node node, String desiredLabel) { + for (LabelAtom label : node.getAssignedLabels()) { + if (label.getExpression().equalsIgnoreCase(desiredLabel)) { + return true; + } + } + return false; + } +} diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly index e41b46992..7a3a97140 100644 --- a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly @@ -39,6 +39,9 @@ THE SOFTWARE. + + + @@ -52,6 +55,15 @@ THE SOFTWARE. + + + + + + + + + diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-instanceTagForJenkins.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-instanceTagForJenkins.html new file mode 100644 index 000000000..d69ec46e1 --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-instanceTagForJenkins.html @@ -0,0 +1 @@ +Instance tag assigned in EC2 that align with the node.getSelfLabel in Jenkins. \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html new file mode 100644 index 000000000..6200becc7 --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html @@ -0,0 +1,3 @@ +This Jenkins marker tag to specify whether or not this plugin should try to find a backend EC2 +instance to stop/start. If blank, the plugin will attempt to stop/start all configured Jenkins nodes. +This can be used to limit which instances this plugin should control. \ No newline at end of file diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-startStopNodes.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-startStopNodes.html new file mode 100644 index 000000000..3ec31eaa6 --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-startStopNodes.html @@ -0,0 +1,3 @@ +Allow the EC2 Plugin to start/stop nodes other than the ones that it created. +This is useful for situations where existing slave instances are present that do not have AMI templates that can be used +to create instances on-demand. \ No newline at end of file diff --git a/src/test/java/hudson/plugins/ec2/InstanceStopTimerTest.java b/src/test/java/hudson/plugins/ec2/InstanceStopTimerTest.java new file mode 100644 index 000000000..031192985 --- /dev/null +++ b/src/test/java/hudson/plugins/ec2/InstanceStopTimerTest.java @@ -0,0 +1,103 @@ +package hudson.plugins.ec2; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +import hudson.model.Computer; +import hudson.model.Executor; +import hudson.model.Node; +import java.io.IOException; +import java.util.Collections; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +public class InstanceStopTimerTest { + + @Rule + public JenkinsRule r = new JenkinsRule(); + + private AmazonEC2Cloud testCloud; + + @Before + public void init() throws Exception { + testCloud = getMockCloud(); + r.jenkins.clouds.add(testCloud); + Node node = getNode(); + r.jenkins.setNodes(Collections.singletonList(node)); + } + + @Test + public void testIdleNodeShouldBeStopped() throws IOException, InterruptedException { + Computer computer = mock(Computer.class); + when(computer.isConnecting()).thenReturn(false); + Executor executor = mock(Executor.class); + when(executor.isIdle()).thenReturn(true); + when(executor.getIdleStartMilliseconds()).thenReturn(0L); + when(computer.getAllExecutors()).thenReturn(Collections.singletonList(executor)); + TestableStopTimer stopTimer = new TestableStopTimer(computer); + stopTimer.execute(null); + verify(testCloud, times(1)).stopNode(any()); + } + + @Test + public void testNoComputer() throws IOException, InterruptedException { + TestableStopTimer stopTimer = new TestableStopTimer(null); + stopTimer.execute(null); + verify(testCloud, times(0)).stopNode(any()); + } + + @Test + public void testNodeIsConnecting() throws IOException, InterruptedException { + Computer computer = mock(Computer.class); + when(computer.isConnecting()).thenReturn(true); + when(computer.isOnline()).thenReturn(false); + TestableStopTimer stopTimer = new TestableStopTimer(computer); + stopTimer.execute(null); + verify(testCloud, times(0)).stopNode(any()); + } + + @Test + public void testNonIdleNodeShouldNotStop() throws IOException, InterruptedException { + Computer computer = mock(Computer.class); + when(computer.isConnecting()).thenReturn(false); + Executor executor = mock(Executor.class); + when(executor.isIdle()).thenReturn(false); + when(executor.getIdleStartMilliseconds()).thenReturn(System.currentTimeMillis()); + when(computer.getAllExecutors()).thenReturn(Collections.singletonList(executor)); + TestableStopTimer stopTimer = new TestableStopTimer(computer); + stopTimer.execute(null); + verify(testCloud, times(0)).stopNode(any()); + } + + private Node getNode() { + Node node = mock(Node.class); + when(node.getNodeName()).thenReturn("Test Node"); + return node; + } + + private AmazonEC2Cloud getMockCloud() { + AmazonEC2Cloud cloud = mock(AmazonEC2Cloud.class); + when(cloud.isStartStopNodes()).thenReturn(true); + when(cloud.getMaxIdleMinutes()).thenReturn("2"); + when(cloud.isEc2Node(any())).thenReturn(true); + return cloud; + } + + private static class TestableStopTimer extends InstanceStopTimer { + private Computer computer; + + public TestableStopTimer(Computer testComputer) { + computer = testComputer; + } + + @Override + protected Computer getComputer(Node node) { + return computer; + } + } +} diff --git a/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java b/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java new file mode 100644 index 000000000..81a77e180 --- /dev/null +++ b/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java @@ -0,0 +1,108 @@ +package hudson.plugins.ec2; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +import hudson.model.Label; +import hudson.model.LoadStatistics.LoadStatisticsSnapshot; +import hudson.model.Node; +import hudson.model.labels.LabelAtom; +import hudson.slaves.NodeProvisioner; +import hudson.slaves.NodeProvisioner.StrategyState; +import java.lang.reflect.Constructor; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; +import org.powermock.core.classloader.annotations.PrepareForTest; + +@PrepareForTest(NodeProvisioner.StrategyState.class) +public class StartInstanceProvisionerStrategyTest { + @Rule + public JenkinsRule r = new JenkinsRule(); + + private AmazonEC2Cloud testCloud; + + private static final String TEST_HOST_LABEL = "testHost"; + + @Before + public void init() throws Exception { + testCloud = getMockCloud(); + r.jenkins.clouds.add(testCloud); + Node node = getNode(); + r.jenkins.setNodes(Collections.singletonList(node)); + } + + @Test + public void testNodeShouldBeStarted() throws Exception { + TestableStartInstanceProvisioner provisioner = new TestableStartInstanceProvisioner(false); + provisioner.apply(getStrategyState(1, 0)); + verify(testCloud, times(1)).startNode(any()); + } + + @Test + public void testNeedMetDontStart() throws Exception { + TestableStartInstanceProvisioner provisioner = new TestableStartInstanceProvisioner(false); + provisioner.apply(getStrategyState(1, 1)); + verify(testCloud, times(0)).startNode(any()); + } + + @Test + public void testNodeIsOnline() throws Exception { + TestableStartInstanceProvisioner provisioner = new TestableStartInstanceProvisioner(true); + provisioner.apply(getStrategyState(1, 0)); + verify(testCloud, times(0)).startNode(any()); + } + + private Node getNode() { + Node node = mock(Node.class); + when(node.getNumExecutors()).thenReturn(0); + when(node.getNodeName()).thenReturn("Test Node"); + Set labels = new HashSet<>(); + LabelAtom ec2Label = new LabelAtom(TEST_HOST_LABEL); + labels.add(ec2Label); + when(node.getAssignedLabels()).thenReturn(labels); + return node; + } + + private AmazonEC2Cloud getMockCloud() { + AmazonEC2Cloud cloud = mock(AmazonEC2Cloud.class); + when(cloud.isStartStopNodes()).thenReturn(true); + when(cloud.getMaxIdleMinutes()).thenReturn("2"); + when(cloud.isEc2Node(any())).thenReturn(true); + return cloud; + } + + private StrategyState getStrategyState(int queueLength, int planned) throws Exception { + Label label = mock(Label.class); + when(label.getExpression()).thenReturn(TEST_HOST_LABEL); + LoadStatisticsSnapshot snapshot = LoadStatisticsSnapshot.builder().withQueueLength(queueLength).build(); + NodeProvisioner nodeProvisioner = new NodeProvisioner(label, null); + NodeProvisioner.StrategyState strategyState = null; + for (Constructor constructor : NodeProvisioner.StrategyState.class.getDeclaredConstructors()) { + if (constructor.getParameterCount() == 4) { + constructor.setAccessible(true); + strategyState = (StrategyState) constructor.newInstance(nodeProvisioner, snapshot, label, planned); + break; + } + } + return strategyState; + } + + private static class TestableStartInstanceProvisioner extends StartInstanceProvisionerStrategy { + private boolean nodeOnline; + + public TestableStartInstanceProvisioner(boolean nodeOnline) { + this.nodeOnline = nodeOnline; + } + @Override protected boolean isNodeOnline(Node node) { + return nodeOnline; + } + } +} From 1ef2a894fc083141b8f52c027aa54a7d5306663d Mon Sep 17 00:00:00 2001 From: Troy Mohl Date: Mon, 16 Mar 2020 15:50:30 -0400 Subject: [PATCH 3/5] Add tag to prevent instance from being stopped --- .../hudson/plugins/ec2/AmazonEC2Cloud.java | 57 ++++++++++++++----- .../hudson/plugins/ec2/InstanceStopTimer.java | 2 +- .../ec2/AmazonEC2Cloud/config-entries.jelly | 5 +- ...gForEc2.html => help-nodeLabelForEc2.html} | 0 .../help-preventStopAwsTag.html | 2 + 5 files changed, 49 insertions(+), 17 deletions(-) rename src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/{help-nodeTagForEc2.html => help-nodeLabelForEc2.html} (100%) create mode 100644 src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html diff --git a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java index 34e4be2ca..0219b34f2 100644 --- a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java @@ -36,6 +36,7 @@ import com.amazonaws.services.ec2.model.Reservation; import com.amazonaws.services.ec2.model.StartInstancesRequest; import com.amazonaws.services.ec2.model.StopInstancesRequest; +import com.amazonaws.services.ec2.model.Tag; import com.google.common.annotations.VisibleForTesting; import hudson.Extension; import hudson.Util; @@ -63,7 +64,6 @@ import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.StaplerResponse; import org.kohsuke.stapler.interceptor.RequirePOST; /** @@ -95,7 +95,9 @@ public class AmazonEC2Cloud extends EC2Cloud { private String instanceTagForJenkins; - private String nodeTagForEc2; + private String nodeLabelForEc2; + + private String preventStopAwsTag; private String maxIdleMinutes; @@ -186,23 +188,32 @@ public void setAltEC2Endpoint(String altEC2Endpoint) { this.altEC2Endpoint = altEC2Endpoint; } - public String getNodeTagForEc2() { - return nodeTagForEc2; + public String getNodeLabelForEc2() { + return nodeLabelForEc2; + } + + @DataBoundSetter + public void setNodeLabelForEc2(String nodeLabelForEc2 ) { + this.nodeLabelForEc2 = nodeLabelForEc2; + } + + public String getPreventStopAwsTag() { + return preventStopAwsTag; } @DataBoundSetter - public void setNodeTagForEc2(String nodeTagForEc2) { - this.nodeTagForEc2 = nodeTagForEc2; + public void setPreventStopAwsTag( String preventStopAwsTag ) { + this.preventStopAwsTag = preventStopAwsTag; } public boolean isEc2Node(Node node) { //If no label is specified then we check all nodes - if (nodeTagForEc2 == null || nodeTagForEc2.trim().length() == 0) { + if ( nodeLabelForEc2 == null || nodeLabelForEc2.trim().length() == 0) { return true; } for (LabelAtom label : node.getAssignedLabels()) { - if (label.getExpression().equalsIgnoreCase(nodeTagForEc2)) { + if (label.getExpression().equalsIgnoreCase( nodeLabelForEc2 )) { return true; } } @@ -281,13 +292,17 @@ public void stopNode(Node node) { final String instanceId = nodeInstance.getInstanceId(); - try { - StopInstancesRequest request = new StopInstancesRequest(); - request.setInstanceIds(Collections.singletonList(instanceId)); - connect().stopInstances(request); - LOGGER.log(Level.INFO, "Stopped instance: {0}", instanceId); - } catch (Exception e) { - LOGGER.log(Level.INFO, "Unable to stop instance: " + instanceId, e); + if (stopAllowed( nodeInstance )) { + try { + StopInstancesRequest request = new StopInstancesRequest(); + request.setInstanceIds( Collections.singletonList( instanceId ) ); + connect().stopInstances( request ); + LOGGER.log( Level.INFO, "Stopped instance: {0}", instanceId ); + } catch ( Exception e ) { + LOGGER.log( Level.INFO, "Unable to stop instance: " + instanceId, e ); + } + } else { + LOGGER.log( Level.FINEST, "Not allowed to stop node: {0}", instanceId); } } @@ -339,6 +354,18 @@ private Instance getInstance(List filters, InstanceStateName desiredStat return null; } + private boolean stopAllowed(Instance instance) { + List tags = instance.getTags(); + if (tags != null) { + for ( Tag tag : tags) { + if (tag.getKey().trim().equals( preventStopAwsTag )) { + return false; + } + } + } + return true; + } + @Extension public static class DescriptorImpl extends EC2Cloud.DescriptorImpl { diff --git a/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java b/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java index 96dd33ebe..d68eb45a8 100644 --- a/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java +++ b/src/main/java/hudson/plugins/ec2/InstanceStopTimer.java @@ -79,7 +79,7 @@ private void stopNode(Node node) { continue; AmazonEC2Cloud ec2 = (AmazonEC2Cloud) cloud; if (ec2.isStartStopNodes() && ec2.isEc2Node(node)) { - LOGGER.log(Level.INFO, "Requesting stop on {0} of {1}", new Object[] {ec2.getCloudName(), node.getNodeName()}); + LOGGER.log(Level.FINE, "Requesting stop on {0} of {1}", new Object[] {ec2.getCloudName(), node.getNodeName()}); try { ec2.stopNode(node); } catch (Exception e) { diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly index 7a3a97140..4a33ce909 100644 --- a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/config-entries.jelly @@ -55,12 +55,15 @@ THE SOFTWARE. - + + + + diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeLabelForEc2.html similarity index 100% rename from src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeTagForEc2.html rename to src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-nodeLabelForEc2.html diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html new file mode 100644 index 000000000..a465ee3a7 --- /dev/null +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html @@ -0,0 +1,2 @@ +Marker tag used on EC2 instances that should not be stopped by this plugin. +Tag value is not checked as the presence of the tag is enough to prevent the plugin from stopping the instance. \ No newline at end of file From 3ab83955fd822f72ac69b5d655ce4346a5f28562 Mon Sep 17 00:00:00 2001 From: Troy Mohl Date: Wed, 18 Mar 2020 12:05:19 -0400 Subject: [PATCH 4/5] prevent_stop tag presence and true, prevents stopping instance --- .../java/hudson/plugins/ec2/AmazonEC2Cloud.java | 16 +++++++++++++++- .../AmazonEC2Cloud/help-preventStopAwsTag.html | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java index 0219b34f2..c639c46e6 100644 --- a/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java +++ b/src/main/java/hudson/plugins/ec2/AmazonEC2Cloud.java @@ -61,6 +61,7 @@ import javax.annotation.Nullable; import javax.servlet.ServletException; import jenkins.model.Jenkins; +import org.apache.commons.lang.BooleanUtils; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; @@ -359,13 +360,26 @@ private boolean stopAllowed(Instance instance) { if (tags != null) { for ( Tag tag : tags) { if (tag.getKey().trim().equals( preventStopAwsTag )) { - return false; + boolean value = isValueTrue( tag.getValue() ); + return !value; } } } return true; } + private boolean isValueTrue(String value) { + boolean boolValue = false; + + if (value == null) { + return false; + } else { + boolValue = BooleanUtils.isTrue( BooleanUtils.toBooleanObject( value ) ); + } + + return boolValue; + } + @Extension public static class DescriptorImpl extends EC2Cloud.DescriptorImpl { diff --git a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html index a465ee3a7..bdabdbb56 100644 --- a/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html +++ b/src/main/resources/hudson/plugins/ec2/AmazonEC2Cloud/help-preventStopAwsTag.html @@ -1,2 +1,2 @@ -Marker tag used on EC2 instances that should not be stopped by this plugin. -Tag value is not checked as the presence of the tag is enough to prevent the plugin from stopping the instance. \ No newline at end of file +Tag used on EC2 instances that should not be stopped by this plugin. +Tag value set to "true" will prevent the plugin from stopping a given instance. If the tag isn't present or value is false, the instance will be stopped. \ No newline at end of file From 39e02464129eb41316057cd106fb861dd6ffd9c6 Mon Sep 17 00:00:00 2001 From: Troy Mohl Date: Thu, 26 Mar 2020 14:24:55 -0400 Subject: [PATCH 5/5] Fix bug where label expressions weren't correctly handled --- .../ec2/StartInstanceProvisionerStrategy.java | 19 ++++++++++--------- .../StartInstanceProvisionerStrategyTest.java | 3 +-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java b/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java index c1050df4b..e438a61e1 100644 --- a/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java +++ b/src/main/java/hudson/plugins/ec2/StartInstanceProvisionerStrategy.java @@ -6,7 +6,6 @@ import hudson.model.Label; import hudson.model.LoadStatistics; import hudson.model.Node; -import hudson.model.labels.LabelAtom; import hudson.slaves.Cloud; import hudson.slaves.NodeProvisioner; import hudson.slaves.NodeProvisioner.PlannedNode; @@ -30,6 +29,10 @@ public class StartInstanceProvisionerStrategy extends NodeProvisioner.Strategy { public NodeProvisioner.StrategyDecision apply(NodeProvisioner.StrategyState strategyState) { final Label label = strategyState.getLabel(); + if (label == null) { + return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES; + } + LOGGER.log(Level.FINEST, "Calling into StartInstanceProvisionerStrategy for label: {0}", label.getExpression()); LoadStatistics.LoadStatisticsSnapshot snapshot = strategyState.getSnapshot(); @@ -44,7 +47,7 @@ public NodeProvisioner.StrategyDecision apply(NodeProvisioner.StrategyState stra Jenkins jenkinsInstance = Jenkins.get(); LOGGER.log(Level.FINE, "Attempting to find node for label: {0}", label); for (Node node : jenkinsInstance.getNodes()) { - if (nodeHasLabel(node, label.getExpression())) { + if (nodeHasLabel(node, label)) { LOGGER.log(Level.FINE,"Found the node, checking if it's running"); if (!isNodeOnline(node)) { LOGGER.log(Level.FINE,"Attempting to start node: {0}", node.getNodeName()); @@ -105,7 +108,7 @@ private int getCurrentCapacity(Label label) { if (isNodeOnline(node)) { Computer computer = node.toComputer(); if (computer != null && computer.isOnline() && !computer.isConnecting()) { - if (nodeHasLabel(node, label.getExpression())) { + if (nodeHasLabel(node, label)) { currentCapacity += node.getNumExecutors(); } } @@ -114,12 +117,10 @@ private int getCurrentCapacity(Label label) { return currentCapacity; } - private boolean nodeHasLabel(Node node, String desiredLabel) { - for (LabelAtom label : node.getAssignedLabels()) { - if (label.getExpression().equalsIgnoreCase(desiredLabel)) { - return true; - } + private boolean nodeHasLabel(Node node, Label desiredLabel) { + if (node == null) { + return false; } - return false; + return desiredLabel.matches( node ); } } diff --git a/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java b/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java index 81a77e180..e6752b0c5 100644 --- a/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java +++ b/src/test/java/hudson/plugins/ec2/StartInstanceProvisionerStrategyTest.java @@ -80,8 +80,7 @@ private AmazonEC2Cloud getMockCloud() { } private StrategyState getStrategyState(int queueLength, int planned) throws Exception { - Label label = mock(Label.class); - when(label.getExpression()).thenReturn(TEST_HOST_LABEL); + Label label = Label.parseExpression( TEST_HOST_LABEL ); LoadStatisticsSnapshot snapshot = LoadStatisticsSnapshot.builder().withQueueLength(queueLength).build(); NodeProvisioner nodeProvisioner = new NodeProvisioner(label, null); NodeProvisioner.StrategyState strategyState = null;