Allow each policy to define its own execution schedule (#434)

* Refactor PolicyExecutorTask infra * so that each policy can define its own execution schedule * a default cron schedule will be applied if none defined * policy id is consistent whether git-backed or not * includes updates to DDL * includes config property for task scheduling pool size * removes cron.execution config property * More updates in support of scheduling individual policies for execution * includes additions and refactoring * polymorphism is our friend * Upgrade Spring Boot version to 3.2.5 * Increase width for git_commit column
cf-toolsuite · May 9, 2024 · 6983bcb · 6983bcb
1 parent 1d0e1bd
commit 6983bcb
Show file tree

Hide file tree

Showing 57 changed files with 571 additions and 241 deletions.
diff --git a/pom.xml b/pom.xml
@@ -53,7 +53,7 @@
   <parent>
     <groupId>org.springframework.boot</groupId>
     <artifactId>spring-boot-starter-parent</artifactId>
-    <version>3.2.4</version>
+    <version>3.2.5</version>
     <relativePath/>
     <!-- lookup parent from repository -->
   </parent>
@@ -609,7 +609,7 @@
       <plugin>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-maven-plugin</artifactId>
-        <version>3.2.4</version>
+        <version>3.2.5</version>
         <configuration>
           <image>
             <builder>paketobuildpacks/builder-jammy-tiny</builder>

diff --git a/samples/application-pcfone.yml b/samples/application-pcfone.yml
@@ -14,7 +14,6 @@ logging:
 # @see https://crontab.guru for help, first parameter is seconds
 cron:
   collection: "0 0 0 * * *"
-  execution: "0 0 2 * * *"
 
 management:
   endpoints:

diff --git a/samples/application-pws.yml b/samples/application-pws.yml
@@ -14,7 +14,6 @@ logging:
 # @see https://crontab.guru for help, first parameter is seconds
 cron:
   collection: "0 0 0 * * *"
-  execution: "0 0 2 * * *"
 
 management:
   endpoints:

diff --git a/samples/secrets.pcfone.json b/samples/secrets.pcfone.json
@@ -5,6 +5,5 @@
     "CF_REFRESH-TOKEN": "xxxxxx",
     "CF_ORGANIZATION-BLACK-LIST": [ "system" ],
     "CF_ACCOUNT-REGEX": "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$",
-    "CRON_COLLECTION": "0 0 0 * * *",
-    "CRON_EXECUTION": "0 0 2 * * *"
+    "CRON_COLLECTION": "0 0 0 * * *"
 }
diff --git a/samples/secrets.pws.json b/samples/secrets.pws.json
@@ -7,7 +7,6 @@
     "CF_ORGANIZATION-BLACK-LIST": [ "system" ],
     "CF_ACCOUNT-REGEX": "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$",
     "CRON_COLLECTION": "0 0 0 * * *",
-    "CRON_EXECUTION": "0 0 2 * * *",
     "CF_POLICIES_GIT_URI": "https://github.com/cf-toolsuite/cf-butler-sample-config.git",
     "CF_POLICIES_GIT_COMMIT": "e745cfe5d93e6517038675fd6b0fe3b85524f130",
     "CF_POLICIES_GIT_FILE-PATHS": [

diff --git a/samples/secrets.pws.with-mysql.json b/samples/secrets.pws.with-mysql.json
@@ -7,7 +7,6 @@
     "CF_ORGANIZATION-BLACK-LIST": [ "system" ],
     "CF_ACCOUNT-REGEX": "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$",
     "CRON_COLLECTION": "0 0 0 * * *",
-    "CRON_EXECUTION": "0 0 2 * * *",
     "CF_POLICIES_GIT_URI": "https://github.com/cf-toolsuite/cf-butler-sample-config.git",
     "CF_POLICIES_GIT_COMMIT": "e745cfe5d93e6517038675fd6b0fe3b85524f130",
     "CF_POLICIES_GIT_FILE-PATHS": [

diff --git a/samples/secrets.pws.with-postgres.json b/samples/secrets.pws.with-postgres.json
@@ -7,7 +7,6 @@
     "CF_ORGANIZATION-BLACK-LIST": [ "system" ],
     "CF_ACCOUNT-REGEX": "^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$",
     "CRON_COLLECTION": "0 0 0 * * *",
-    "CRON_EXECUTION": "0 0 2 * * *",
     "CF_POLICIES_GIT_URI": "https://github.com/cf-toolsuite/cf-butler-sample-config.git",
     "CF_POLICIES_GIT_COMMIT": "e745cfe5d93e6517038675fd6b0fe3b85524f130",
     "CF_POLICIES_GIT_FILE-PATHS": [

diff --git a/src/main/java/org/cftoolsuite/cfapp/AppInit.java b/src/main/java/org/cftoolsuite/cfapp/AppInit.java
@@ -3,14 +3,12 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
-import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.transaction.annotation.EnableTransactionManagement;
 
 import io.netty.util.ResourceLeakDetector;
 import reactor.core.publisher.Hooks;
 
 
-@EnableScheduling
 @EnableTransactionManagement
 @ConfigurationPropertiesScan
 @SpringBootApplication

diff --git a/src/main/java/org/cftoolsuite/cfapp/config/ButlerCfEnvProcessor.java b/src/main/java/org/cftoolsuite/cfapp/config/ButlerCfEnvProcessor.java
@@ -79,7 +79,6 @@ public void process(CfCredentials cfCredentials, Map<String, Object> properties)
         addOrUpdatePropertyValue("pivnet.apiToken", "PIVNET_API-TOKEN", cfCredentials, properties);
         addOrUpdatePropertyValue("pivnet.enabled", "PIVNET_ENABLED", cfCredentials, properties);
         addOrUpdatePropertyValue("cron.collection", "CRON_COLLECTION", cfCredentials, properties);
-        addOrUpdatePropertyValue("cron.execution", "CRON_EXECUTION", cfCredentials, properties);
         addOrUpdatePropertyValue("management.endpoints.web.exposure.include", "EXPOSED_ACTUATOR_ENDPOINTS", cfCredentials, properties);
         addOrUpdatePropertyValue("java.artifacts.fetch.mode", "JAVA_ARTIFACTS_FETCH_MODE", cfCredentials, properties);
     }

diff --git a/src/main/java/org/cftoolsuite/cfapp/controller/OnDemandPolicyTriggerController.java b/src/main/java/org/cftoolsuite/cfapp/controller/OnDemandPolicyTriggerController.java
@@ -1,35 +1,42 @@
 package org.cftoolsuite.cfapp.controller;
 
-import java.util.Collection;
-import java.util.Map;
-
+import org.cftoolsuite.cfapp.service.PoliciesService;
 import org.cftoolsuite.cfapp.task.PolicyExecutorTask;
-import org.springframework.beans.factory.ListableBeanFactory;
+import org.springframework.beans.factory.BeanFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Profile;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 @Profile("on-demand")
 @RestController
 public class OnDemandPolicyTriggerController {
 
-    private ListableBeanFactory factory;
+    private BeanFactory factory;
+    private PoliciesService service;
 
     @Autowired
-    public OnDemandPolicyTriggerController(ListableBeanFactory factory) {
+    public OnDemandPolicyTriggerController(BeanFactory factory, PoliciesService service) {
         this.factory = factory;
+        this.service = service;
     }
 
     @PostMapping("/policies/execute")
-    public Mono<ResponseEntity<Void>> triggerPolicyExection() {
-        Map<String, PolicyExecutorTask> taskMap = factory.getBeansOfType(PolicyExecutorTask.class);
-        Collection<PolicyExecutorTask> tasks = taskMap.values();
-        tasks.forEach(PolicyExecutorTask::execute);
-        return Mono.just(ResponseEntity.accepted().build());
+    public Mono<ResponseEntity<Void>> triggerPolicyExecution() {
+        return service.getTaskMap()
+            .flatMapMany(taskTypeMap ->
+                Flux.fromIterable(taskTypeMap.entrySet())
+                    .flatMap(entry -> {
+                        String policyId = entry.getKey();
+                        Class<? extends PolicyExecutorTask> taskClass = entry.getValue();
+                        PolicyExecutorTask task = factory.getBean(taskClass);
+                        return Mono.fromRunnable(() -> task.execute(policyId));
+                    }))
+            .then(Mono.just(ResponseEntity.accepted().build()));
     }
 
 }
diff --git a/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationOperation.java b/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationOperation.java
@@ -1,8 +1,15 @@
 package org.cftoolsuite.cfapp.domain;
 
 import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.Map;
 import java.util.stream.Collectors;
 
+import org.cftoolsuite.cfapp.task.DeleteAppPolicyExecutorTask;
+import org.cftoolsuite.cfapp.task.PolicyExecutorTask;
+import org.cftoolsuite.cfapp.task.ScaleAppInstancesPolicyExecutorTask;
+import org.cftoolsuite.cfapp.task.StackChangeAppInstancesPolicyExecutorTask;
+import org.cftoolsuite.cfapp.task.StopAppPolicyExecutorTask;
 import org.springframework.util.Assert;
 
 import com.fasterxml.jackson.annotation.JsonValue;
@@ -14,21 +21,34 @@ public enum ApplicationOperation {
     STOP("stop"),
     CHANGE_STACK("change-stack");
 
+    private final String name;
+
+    ApplicationOperation(String name) {
+        this.name = name;
+    }
+
+    static final Map<ApplicationOperation, Class<? extends PolicyExecutorTask>> operationTaskMap = new EnumMap<>(ApplicationOperation.class);
+    static {
+        operationTaskMap.put(ApplicationOperation.DELETE, DeleteAppPolicyExecutorTask.class);
+        operationTaskMap.put(ApplicationOperation.SCALE_INSTANCES, ScaleAppInstancesPolicyExecutorTask.class);
+        operationTaskMap.put(ApplicationOperation.STOP, StopAppPolicyExecutorTask.class);
+        operationTaskMap.put(ApplicationOperation.CHANGE_STACK, StackChangeAppInstancesPolicyExecutorTask.class);
+    }
+
     public static ApplicationOperation from(String name) {
         Assert.hasText(name, "ApplicationOperation must not be null or empty");
         ApplicationOperation result = Arrays.asList(ApplicationOperation.values()).stream().filter(s -> s.getName().equalsIgnoreCase(name)).collect(Collectors.toList()).get(0);
         Assert.notNull(result, String.format("Invalid ApplicationOperation, name=%s", name));
         return result;
     }
 
-    private final String name;
-
-    ApplicationOperation(String name) {
-        this.name = name;
+    public static Class<? extends PolicyExecutorTask> getTaskType(String op) {
+        return operationTaskMap.get(from(op));
     }
 
     @JsonValue
     public String getName() {
         return name;
     }
+
 }
diff --git a/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicy.java b/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicy.java
@@ -27,15 +27,16 @@
 
 @Builder
 @JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ "id", "operation", "description", "state", "options", "organization-whitelist" })
+@JsonPropertyOrder({ "id", "git-commit", "operation", "description", "state", "options", "organization-whitelist", "cron-expression" })
 @Getter
 @ToString
 @Table("application_policy")
-public class ApplicationPolicy implements HasOrganizationWhiteList {
+public class ApplicationPolicy implements HasOrganizationWhiteList, Policy {
 
     public static ApplicationPolicy seed(ApplicationPolicy policy) {
         return ApplicationPolicy
                 .builder()
+                .cronExpression(policy.getCronExpression())
                 .description(policy.getDescription())
                 .operation(policy.getOperation())
                 .options(policy.getOptions())
@@ -44,10 +45,11 @@ public static ApplicationPolicy seed(ApplicationPolicy policy) {
                 .build();
     }
 
-    public static ApplicationPolicy seedWith(ApplicationPolicy policy, String id) {
+    public static ApplicationPolicy seedWith(ApplicationPolicy policy, String gitCommit) {
         return ApplicationPolicy
                 .builder()
-                .id(id)
+                .gitCommit(gitCommit)
+                .cronExpression(policy.getCronExpression())
                 .description(policy.getDescription())
                 .operation(policy.getOperation())
                 .options(policy.getOptions())
@@ -64,6 +66,10 @@ public static ApplicationPolicy seedWith(ApplicationPolicy policy, String id) {
     @JsonProperty("id")
     private String id = Generators.timeBasedGenerator().generate().toString();
 
+    @JsonProperty("git-commit")
+    @Column("git_commit")
+    private String gitCommit;
+
     @JsonProperty("operation")
     private String operation;
 
@@ -82,22 +88,30 @@ public static ApplicationPolicy seedWith(ApplicationPolicy policy, String id) {
     @Column("organization_whitelist")
     private Set<String> organizationWhiteList = new HashSet<>();
 
+    @JsonProperty("cron-expression")
+    @Column("cron_expression")
+    private String cronExpression;
+
     @JsonCreator
     ApplicationPolicy(
             @JsonProperty("pk") Long pk,
             @JsonProperty("id") String id,
+            @JsonProperty("git-commit") String gitCommit,
             @JsonProperty("operation") String operation,
             @JsonProperty("description") String description,
             @JsonProperty("state") String state,
             @JsonProperty("options") Map<String, Object> options,
-            @JsonProperty("organization-whitelist") Set<String> organizationWhiteList) {
+            @JsonProperty("organization-whitelist") Set<String> organizationWhiteList,
+            @JsonProperty("cron-expression") String cronExpression) {
         this.pk = pk;
         this.id = id;
+        this.gitCommit = gitCommit;
         this.operation = operation;
         this.description = description;
         this.state = state;
         this.options = options;
         this.organizationWhiteList = organizationWhiteList;
+        this.cronExpression = cronExpression;
     }
 
     @JsonIgnore
@@ -110,6 +124,10 @@ public <T> T getOption(String key, Class<T> type) {
         return type.cast(value);
     }
 
+    public String getCronExpression() {
+        return StringUtils.isBlank(cronExpression) ? defaultCronExpression(): cronExpression;
+    }
+
     public Map<String, Object> getOptions() {
         return CollectionUtils.isEmpty(options) ? new HashMap<>(): Collections.unmodifiableMap(options);
     }

diff --git a/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicyReadConverter.java b/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicyReadConverter.java
@@ -26,10 +26,12 @@ public ApplicationPolicy convert(Row source) {
                 .builder()
                 .pk(source.get("pk", Long.class))
                 .id(source.get("id", String.class))
+                .gitCommit(source.get("git_commit", String.class))
                 .operation(source.get("operation", String.class))
                 .description(source.get("description", String.class))
                 .options(readOptions(source.get("options", String.class) == null ? "{}" : source.get("options", String.class)))
                 .organizationWhiteList(CsvUtil.parse(source.get("organization_whitelist", String.class)))
+                .cronExpression(source.get("cron_expression", String.class))
                 .state(source.get("state", String.class))
                 .build();
     }

diff --git a/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicyWriteConverter.java b/src/main/java/org/cftoolsuite/cfapp/domain/ApplicationPolicyWriteConverter.java
@@ -23,10 +23,12 @@ public class ApplicationPolicyWriteConverter implements Converter<ApplicationPol
     public OutboundRow convert(ApplicationPolicy source) {
         OutboundRow row = new OutboundRow();
         row.put("id", Parameter.fromOrEmpty(source.getId(), String.class));
+        row.put("git_commit", Parameter.fromOrEmpty(source.getGitCommit(), String.class));
         row.put("operation", Parameter.fromOrEmpty(source.getOperation(), String.class));
         row.put("description", Parameter.fromOrEmpty(source.getDescription(), String.class));
         row.put("state", Parameter.fromOrEmpty(source.getState(), String.class));
         row.put("organization_whitelist", Parameter.fromOrEmpty(source.getOrganizationWhiteList().stream().filter(StringUtils::isNotBlank).collect(Collectors.joining(",")), String.class));
+        row.put("cron_expression", Parameter.fromOrEmpty(source.getCronExpression(), String.class));
         row.put("options", Parameter.fromOrEmpty(CollectionUtils.isEmpty(source.getOptions()) ? null : writeOptions(source.getOptions()), String.class));
         return row;
     }