fix critical services issue on pg follower restart (#8548)

* fix critical services issue on pg follower restart. Directly send data to pg leader from pg-gateway Signed-off-by: Vivek Shankar <vshankar@progress.com> * added health check retry Signed-off-by: Vivek Shankar <vshankar@progress.com> * updated working patch health check config for provided services Signed-off-by: Vivek Shankar <vshankar@progress.com> * renamed folder Signed-off-by: Vivek Shankar <vshankar@progress.com> * review comments Signed-off-by: punitmundra <pmundra@progress.com> * fail timout added in es, changes port from 7432 to 5432 Signed-off-by: Vivek Shankar <vshankar@progress.com> * added log in cs health hook Signed-off-by: Vivek Shankar <vshankar@progress.com> * path after upgrade Signed-off-by: Vivek Shankar <vshankar@progress.com> * added max conn configurable Signed-off-by: Vivek Shankar <vshankar@progress.com> * fix policy Signed-off-by: Vivek Shankar <vshankar@progress.com> * fix policy Signed-off-by: Vivek Shankar <vshankar@progress.com> * revert pg Signed-off-by: Vivek Shankar <vshankar@progress.com> --------- Signed-off-by: Vivek Shankar <vshankar@progress.com> Signed-off-by: punitmundra <pmundra@progress.com> Co-authored-by: punitmundra <pmundra@progress.com>
chef · Sep 13, 2024 · 34eb900 · 34eb900
1 parent c487d62
commit 34eb900
Show file tree

Hide file tree

Showing 27 changed files with 429 additions and 240 deletions.
diff --git a/api/config/deployment/config_request.go b/api/config/deployment/config_request.go
@@ -27,6 +27,7 @@ func NewConfigRequest() *ConfigRequest {
 			},
 			Svc: &ConfigRequest_V1_Service{
 				AdminUser: &ConfigRequest_V1_AdminUser{},
+				Health:    &ConfigRequest_V1_Health{},
 			},
 		},
 	}
@@ -48,6 +49,9 @@ func DefaultConfigRequest() *ConfigRequest {
 	c.V1.Svc.ManifestCacheExpiry = w.String("60m")
 	c.V1.Svc.PackageCleanupMode = w.String("conservative")
 
+	c.V1.Svc.Health.HealthCheckInterval = w.Int32(30)
+	c.V1.Svc.Health.Services = []string{}
+
 	return c
 }
 

diff --git a/api/config/deployment/config_request.pb.go b/api/config/deployment/config_request.pb.go
diff --git a/api/config/deployment/config_request.proto b/api/config/deployment/config_request.proto
@@ -71,6 +71,13 @@ message ConfigRequest {
 
 			google.protobuf.BoolValue enable_dev_monitoring = 15;
 			repeated string products = 16;
+
+			Health health = 17;
+		}
+
+		message Health {
+			google.protobuf.Int32Value health_check_interval = 1;
+			repeated string services = 2;
 		}
 
 		message AdminUser {

diff --git a/api/config/esgateway/config_request.go b/api/config/esgateway/config_request.go
@@ -51,6 +51,7 @@ func DefaultConfigRequest() *ConfigRequest {
 
 	c.V1.Sys.Ngx.Main.WorkerProcesses = w.Int32(4)
 	c.V1.Sys.Ngx.Main.MaxFails = w.Int32(10)
+	c.V1.Sys.Ngx.Main.FailTimeout = w.Int32(10)
 	// TODO(ssd) 2020-12-08: Enable automatic discovery of system
 	// resolvers once we are a bit more confident in the nginx
 	// module we are using to do dynamic resolution.

diff --git a/api/config/esgateway/config_request.pb.go b/api/config/esgateway/config_request.pb.go
diff --git a/api/config/esgateway/config_request.proto b/api/config/esgateway/config_request.proto
@@ -45,6 +45,7 @@ message ConfigRequest {
 					google.protobuf.Int32Value max_fails = 2;
 					reserved 3;
 					Resolvers resolvers = 4;
+					google.protobuf.Int32Value fail_timeout = 5;
 
 					message Resolvers {
 						google.protobuf.BoolValue enable_system_nameservers = 1;

diff --git a/api/config/pg_gateway/config_request.go b/api/config/pg_gateway/config_request.go
@@ -40,6 +40,7 @@ func DefaultConfigRequest() *ConfigRequest {
 	c.V1.Sys.Timeouts.Connect = w.Int32(5)
 	c.V1.Sys.Timeouts.Idle = w.Int32(43200)
 	c.V1.Sys.Resolvers.EnableSystemNameservers = w.Bool(false)
+	c.V1.Sys.MaxConnections = w.Int32(350)
 
 	return c
 }