fix(engine): adjust container readiness health checks (#425)

agneum · agneum · commit 7aa8e56e9a74 · 2022-09-14T02:45:34.000Z
diff --git a/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go b/engine/internal/retrieval/engine/postgres/tools/health/healthcheck.go
@@ -46,21 +46,21 @@ func GetConfig(username, dbname string, options ...ContainerOption) *container.H
 	return healthConfig
 }
 
-// OptionRetries allows overwrite retries counter.
+// OptionRetries allows overwriting retries counter.
 func OptionRetries(retries int) ContainerOption {
 	return func(h *container.HealthConfig) {
 		h.Retries = retries
 	}
 }
 
-// OptionInterval allows overwrite a health check interval.
+// OptionInterval allows overwriting a health check interval.
 func OptionInterval(interval time.Duration) ContainerOption {
 	return func(h *container.HealthConfig) {
 		h.Interval = interval
 	}
 }
 
-// OptionTest allows overwrite a health check test command.
+// OptionTest allows overwriting a health check test command.
 func OptionTest(testCommand string) ContainerOption {
 	return func(h *container.HealthConfig) {
 		if testCommand != "" {
diff --git a/engine/internal/retrieval/engine/postgres/tools/tools.go b/engine/internal/retrieval/engine/postgres/tools/tools.go
@@ -321,6 +321,8 @@ func StopPostgres(ctx context.Context, dockerClient *client.Client, containerID,
 func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, containerID string) (err error) {
 	log.Msg("Check container readiness: ", containerID)
 
+	var errorRepeats bool
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -344,11 +346,22 @@ func CheckContainerReadiness(ctx context.Context, dockerClient *client.Client, c
 			}
 
 			if healthCheckLength := len(resp.State.Health.Log); healthCheckLength > 0 {
+				// Checking exit code 2 and 3 because pg_isready returns
+				//  0 to the shell if the server is accepting connections normally,
+				//  1 if the server is rejecting connections (for example during startup),
+				//  2 if there was no response to the connection attempt, and
+				//  3 if no attempt was made (for example due to invalid parameters).
+				// Supposedly, the status 2 will be returned in cases where the server is not running
+				// and will not start on its own, so there is no reason to wait for all specified retries.
 				if lastHealthCheck := resp.State.Health.Log[healthCheckLength-1]; lastHealthCheck.ExitCode > 1 {
-					return &ErrHealthCheck{
-						ExitCode: lastHealthCheck.ExitCode,
-						Output:   lastHealthCheck.Output,
+					if errorRepeats {
+						return &ErrHealthCheck{
+							ExitCode: lastHealthCheck.ExitCode,
+							Output:   lastHealthCheck.Output,
+						}
 					}
+
+					errorRepeats = true
 				}
 			}
 		}

Original file line number	Diff line number	Diff line change
`@@ -46,21 +46,21 @@ func GetConfig(username, dbname string, options ...ContainerOption) *container.H`
`46`	`46`	`return healthConfig`
`47`	`47`	`}`
`48`	`48`
`49`		`-// OptionRetries allows overwrite retries counter.`
	`49`	`+// OptionRetries allows overwriting retries counter.`
`50`	`50`	`func OptionRetries(retries int) ContainerOption {`
`51`	`51`	`return func(h *container.HealthConfig) {`
`52`	`52`	`h.Retries = retries`
`53`	`53`	`}`
`54`	`54`	`}`
`55`	`55`
`56`		`-// OptionInterval allows overwrite a health check interval.`
	`56`	`+// OptionInterval allows overwriting a health check interval.`
`57`	`57`	`func OptionInterval(interval time.Duration) ContainerOption {`
`58`	`58`	`return func(h *container.HealthConfig) {`
`59`	`59`	`h.Interval = interval`
`60`	`60`	`}`
`61`	`61`	`}`
`62`	`62`
`63`		`-// OptionTest allows overwrite a health check test command.`
	`63`	`+// OptionTest allows overwriting a health check test command.`
`64`	`64`	`func OptionTest(testCommand string) ContainerOption {`
`65`	`65`	`return func(h *container.HealthConfig) {`
`66`	`66`	`if testCommand != "" {`