Merge branch '425-health-check-attempts' into 'master'

agneum · agneum · commit 218244c08d83 · 2022-09-16T05:43:29.000Z
fix(engine): increase health check attempts (#425)

Closes #425

See merge request postgres-ai/database-lab!602
diff --git a/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go b/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go
@@ -17,7 +17,9 @@ const (
 	hcInterval    = 5 * time.Second
 	hcTimeout     = 2 * time.Second
 	hcStartPeriod = 3 * time.Second
-	hcRetries     = 15
+
+	// HCRetries defines the default number of retries.
+	HCRetries = 15
 
 	// DefaultRestoreInterval defines a default health check interval for database restoration.
 	DefaultRestoreInterval = 5 * time.Second
@@ -36,7 +38,7 @@ func GetConfig(username, dbname string, options ...ContainerOption) *container.H
 		Interval:    hcInterval,
 		Timeout:     hcTimeout,
 		StartPeriod: hcStartPeriod,
-		Retries:     hcRetries,
+		Retries:     HCRetries,
 	}
 
 	for _, healthCheckOption := range options {
diff --git a/engine/internal/retrieval/engine/postgres/tools/tools.go b/engine/internal/retrieval/engine/postgres/tools/tools.go
@@ -35,6 +35,7 @@ import (
 	"github.com/shirou/gopsutil/host"
 
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/defaults"
+	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/health"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
 )
 
@@ -321,7 +322,7 @@ func StopPostgres(ctx context.Context, dockerClient *client.Client, containerID,
 func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, containerID string) (err error) {
 	log.Msg("Check container readiness: ", containerID)
 
-	var errorRepeats bool
+	var errorRepeats int
 
 	for {
 		select {
@@ -354,14 +355,14 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c
 				// Supposedly, the status 2 will be returned in cases where the server is not running
 				// and will not start on its own, so there is no reason to wait for all specified retries.
 				if lastHealthCheck := resp.State.Health.Log[healthCheckLength-1]; lastHealthCheck.ExitCode > 1 {
-					if errorRepeats {
+					if errorRepeats >= health.HCRetries {
 						return &ErrHealthCheck{
 							ExitCode: lastHealthCheck.ExitCode,
 							Output:   lastHealthCheck.Output,
 						}
 					}
 
-					errorRepeats = true
+					errorRepeats++
 				}
 			}
 		}

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ import (`
`35`	`35`	`"github.com/shirou/gopsutil/host"`
`36`	`36`
`37`	`37`	`"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/defaults"`
	`38`	`+ "gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/health"`
`38`	`39`	`"gitlab.com/postgres-ai/database-lab/v3/pkg/log"`
`39`	`40`	`)`
`40`	`41`
`@@ -321,7 +322,7 @@ func StopPostgres(ctx context.Context, dockerClient *client.Client, containerID,`
`321`	`322`	`func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, containerID string) (err error) {`
`322`	`323`	`log.Msg("Check container readiness: ", containerID)`
`323`	`324`
`324`		`- var errorRepeats bool`
	`325`	`+ var errorRepeats int`
`325`	`326`
`326`	`327`	`for {`
`327`	`328`	`select {`
`@@ -354,14 +355,14 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c`
`354`	`355`	`// Supposedly, the status 2 will be returned in cases where the server is not running`
`355`	`356`	`// and will not start on its own, so there is no reason to wait for all specified retries.`
`356`	`357`	`if lastHealthCheck := resp.State.Health.Log[healthCheckLength-1]; lastHealthCheck.ExitCode > 1 {`
`357`		`- if errorRepeats {`
	`358`	`+ if errorRepeats >= health.HCRetries {`
`358`	`359`	`return &ErrHealthCheck{`
`359`	`360`	`ExitCode: lastHealthCheck.ExitCode,`
`360`	`361`	`Output: lastHealthCheck.Output,`
`361`	`362`	`}`
`362`	`363`	`}`
`363`	`364`
`364`		`- errorRepeats = true`
	`365`	`+ errorRepeats++`
`365`	`366`	`}`
`366`	`367`	`}`
`367`	`368`	`}`