Skip to content

Commit aea3573

Browse files
agneumagneum
committed
feat: create the initial snapshot (physical case) (#140)
1 parent 18ee555 commit aea3573

File tree

25 files changed

+861
-188
lines changed

25 files changed

+861
-188
lines changed

‎cmd/database-lab/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func main() {
8383
}
8484

8585
// Create a new retrieval service to prepare a data directory and start snapshotting.
86-
retrievalSvc, err := retrieval.New(cfg, dockerCLI, provisionSvc)
86+
retrievalSvc, err := retrieval.New(cfg, dockerCLI, provisionSvc.ThinCloneManager())
8787
if err != nil {
8888
log.Fatal("Failed to build a retrieval service:", err)
8989
}

‎configs/config.sample.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,12 @@ retrieval:
183183
- name: logical-snapshot
184184
options:
185185
# It is possible to define a pre-precessing script.
186-
# preprocessingScript: "/tmp/scripts/custom.sh"
186+
# preprocessingScript: "/tmp/scripts/custom.sh"
187+
188+
- name: physical-snapshot
189+
options:
190+
promote: true
191+
# It is possible to define a pre-precessing script.
192+
# preprocessingScript: "/tmp/scripts/custom.sh"
187193

188194
debug: true

‎pkg/retrieval/dbmarker/dbmarker.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,9 @@ const (
3838

3939
// LogicalDataType defines a logical data type.
4040
LogicalDataType = "logical"
41-
//PhysicalDataType = "physical"
41+
42+
// PhysicalDataType defines a physical data type.
43+
PhysicalDataType = "physical"
4244
)
4345

4446
// Init inits DB marker for the data directory.

‎pkg/retrieval/engine/engine.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@ import (
1212
"gitlab.com/postgres-ai/database-lab/pkg/config"
1313
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/components"
1414
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres"
15-
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
15+
"gitlab.com/postgres-ai/database-lab/pkg/services/provision/thinclones"
1616
)
1717

1818
// StageBuilder provides a new stage builder.
19-
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client, prov provision.Provision) (components.StageBuilder, error) {
19+
func StageBuilder(globalCfg *config.Global, dockerCli *client.Client,
20+
cloneManager thinclones.Manager) (components.StageBuilder, error) {
2021
switch globalCfg.Engine {
2122
case postgres.EngineType:
22-
return postgres.NewStageBuilder(globalCfg, dockerCli, prov), nil
23+
return postgres.NewStageBuilder(globalCfg, dockerCli, cloneManager), nil
2324

2425
default:
2526
return nil, errors.New("failed to get engine")

‎pkg/retrieval/engine/postgres/initialize/logical/dump.go

Lines changed: 11 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
package logical
66

77
import (
8-
"bytes"
98
"context"
109
"fmt"
1110
"io"
@@ -15,20 +14,20 @@ import (
1514
"strings"
1615
"time"
1716

18-
"github.com/AlekSi/pointer"
1917
"github.com/docker/docker/api/types"
2018
"github.com/docker/docker/api/types/container"
2119
"github.com/docker/docker/api/types/mount"
2220
"github.com/docker/docker/api/types/network"
2321
"github.com/docker/docker/client"
24-
"github.com/docker/docker/pkg/stdcopy"
2522
"github.com/pkg/errors"
2623

2724
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2825
"gitlab.com/postgres-ai/database-lab/pkg/log"
2926
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
3027
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
3128
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
29+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/defaults"
30+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/health"
3231
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
3332
)
3433

@@ -37,9 +36,8 @@ const (
3736
DumpJobType = "logical-dump"
3837

3938
// Defines dump options.
40-
dumpContainerName = "retrieval_logical_dump"
41-
dumpContainerDir = "/tmp"
42-
dumpContainerStopTimeout = 10 * time.Second
39+
dumpContainerName = "retrieval_logical_dump"
40+
dumpContainerDir = "/tmp"
4341

4442
// Defines dump source types.
4543
sourceTypeLocal = "local"
@@ -159,11 +157,11 @@ Either set 'numberOfJobs' equals to 1 or disable the restore section`)
159157
func (d *DumpJob) setDefaults() {
160158
// TODO: Default yaml values in tags.
161159
if d.DumpOptions.Source.Connection.Port == 0 {
162-
d.DumpOptions.Source.Connection.Port = defaultPort
160+
d.DumpOptions.Source.Connection.Port = defaults.Port
163161
}
164162

165163
if d.DumpOptions.Source.Connection.Username == "" {
166-
d.DumpOptions.Source.Connection.Username = defaultUsername
164+
d.DumpOptions.Source.Connection.Username = defaults.Username
167165
}
168166

169167
if d.DumpOptions.ParallelJobs == 0 {
@@ -222,7 +220,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
222220
&container.Config{
223221
Env: d.getEnvironmentVariables(),
224222
Image: d.DockerImage,
225-
Healthcheck: getContainerHealthConfig(),
223+
Healthcheck: health.GetConfig(),
226224
},
227225
&container.HostConfig{
228226
Mounts: d.getMountVolumes(),
@@ -237,21 +235,7 @@ func (d *DumpJob) Run(ctx context.Context) error {
237235
return errors.Wrap(err, "failed to create container")
238236
}
239237

240-
defer func() {
241-
if err := d.dockerClient.ContainerStop(ctx, cont.ID, pointer.ToDuration(dumpContainerStopTimeout)); err != nil {
242-
log.Err("Failed to stop a dump container: ", err)
243-
}
244-
245-
if err := d.dockerClient.ContainerRemove(ctx, cont.ID, types.ContainerRemoveOptions{
246-
Force: true,
247-
}); err != nil {
248-
log.Err("Failed to remove container: ", err)
249-
250-
return
251-
}
252-
253-
log.Msg(fmt.Sprintf("Stop container: %s. ID: %v", dumpContainerName, cont.ID))
254-
}()
238+
defer tools.RemoveContainer(ctx, d.dockerClient, cont.ID, tools.StopTimeout)
255239

256240
if err := d.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
257241
return errors.Wrap(err, "failed to start container")
@@ -341,34 +325,7 @@ func (d *DumpJob) performDumpCommand(ctx context.Context, cmdOutput io.Writer, c
341325
}
342326
defer execAttach.Close()
343327

344-
// read the cmd output
345-
var errBuf bytes.Buffer
346-
347-
outputDone := make(chan error)
348-
349-
go func() {
350-
// StdCopy de-multiplexes the stream into two writers.
351-
_, err = stdcopy.StdCopy(cmdOutput, &errBuf, execAttach.Reader)
352-
outputDone <- err
353-
}()
354-
355-
select {
356-
case err := <-outputDone:
357-
if err != nil {
358-
return errors.Wrap(err, "failed to copy output")
359-
}
360-
361-
break
362-
363-
case <-ctx.Done():
364-
return ctx.Err()
365-
}
366-
367-
if errBuf.Len() > 0 {
368-
return errors.New(errBuf.String())
369-
}
370-
371-
return nil
328+
return tools.ProcessAttachResponse(ctx, execAttach.Reader, cmdOutput)
372329
}
373330

374331
func (d *DumpJob) getDumpContainerPath() string {
@@ -381,7 +338,7 @@ func (d *DumpJob) getEnvironmentVariables() []string {
381338
"POSTGRES_HOST_AUTH_METHOD=trust",
382339
}
383340

384-
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaultPort {
341+
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaults.Port {
385342
envs = append(envs, "PGPORT="+strconv.Itoa(reservePort))
386343
}
387344

@@ -467,7 +424,7 @@ func (d *DumpJob) buildLogicalDumpCommand() []string {
467424
}
468425

469426
func (d *DumpJob) buildLogicalRestoreCommand() []string {
470-
restoreCmd := []string{"|", "pg_restore", "-U", defaultUsername, "-C", "-d", defaultDBName, "--no-privileges"}
427+
restoreCmd := []string{"|", "pg_restore", "-U", defaults.Username, "-C", "-d", defaults.DBName, "--no-privileges"}
471428

472429
if d.Restore.ForceInit {
473430
restoreCmd = append(restoreCmd, "--clean", "--if-exists")

‎pkg/retrieval/engine/postgres/initialize/logical/logical.go

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,32 +4,3 @@
44

55
// Package logical provides jobs for logical initial operations.
66
package logical
7-
8-
import (
9-
"time"
10-
11-
"github.com/docker/docker/api/types/container"
12-
)
13-
14-
const (
15-
// Default values.
16-
defaultPort = 5432
17-
defaultUsername = "postgres"
18-
defaultDBName = "postgres"
19-
20-
// Defines container health check options.
21-
hcInterval = 5 * time.Second
22-
hcTimeout = 2 * time.Second
23-
hcStartPeriod = 3 * time.Second
24-
hcRetries = 5
25-
)
26-
27-
func getContainerHealthConfig() *container.HealthConfig {
28-
return &container.HealthConfig{
29-
Test: []string{"CMD-SHELL", "pg_isready -U " + defaultUsername},
30-
Interval: hcInterval,
31-
Timeout: hcTimeout,
32-
StartPeriod: hcStartPeriod,
33-
Retries: hcRetries,
34-
}
35-
}

‎pkg/retrieval/engine/postgres/initialize/logical/restore.go

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111
"strconv"
1212
"strings"
1313

14-
"github.com/AlekSi/pointer"
1514
"github.com/docker/docker/api/types"
1615
"github.com/docker/docker/api/types/container"
1716
"github.com/docker/docker/api/types/mount"
@@ -24,6 +23,8 @@ import (
2423
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
2524
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
2625
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
26+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/defaults"
27+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools/health"
2728
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
2829
)
2930

@@ -121,7 +122,7 @@ func (r *RestoreJob) Run(ctx context.Context) error {
121122
"PGDATA=" + pgDataContainerDir,
122123
},
123124
Image: r.RestoreOptions.DockerImage,
124-
Healthcheck: getContainerHealthConfig(),
125+
Healthcheck: health.GetConfig(),
125126
},
126127
&container.HostConfig{
127128
Mounts: []mount.Mount{
@@ -144,21 +145,7 @@ func (r *RestoreJob) Run(ctx context.Context) error {
144145
return errors.Wrap(err, "failed to create container")
145146
}
146147

147-
defer func() {
148-
if err := r.dockerClient.ContainerStop(ctx, cont.ID, pointer.ToDuration(dumpContainerStopTimeout)); err != nil {
149-
log.Err("Failed to stop a dump container: ", err)
150-
}
151-
152-
if err := r.dockerClient.ContainerRemove(ctx, cont.ID, types.ContainerRemoveOptions{
153-
Force: true,
154-
}); err != nil {
155-
log.Err("Failed to remove container: ", err)
156-
157-
return
158-
}
159-
160-
log.Msg(fmt.Sprintf("Stop container: %s. ID: %v", restoreContainerName, cont.ID))
161-
}()
148+
defer tools.RemoveContainer(ctx, r.dockerClient, cont.ID, tools.StopTimeout)
162149

163150
if err := r.dockerClient.ContainerStart(ctx, cont.ID, types.ContainerStartOptions{}); err != nil {
164151
return errors.Wrap(err, "failed to start a container")
@@ -251,10 +238,10 @@ func (r *RestoreJob) retrieveDataStateAt(ctx context.Context, contID string) (st
251238
}
252239

253240
func (r *RestoreJob) buildLogicalRestoreCommand() []string {
254-
restoreCmd := []string{"pg_restore", "-U", defaultUsername, "-C"}
241+
restoreCmd := []string{"pg_restore", "-U", defaults.Username, "-C"}
255242

256243
if r.ForceInit {
257-
restoreCmd = append(restoreCmd, "-d", defaultDBName, "--clean", "--if-exists")
244+
restoreCmd = append(restoreCmd, "-d", defaults.DBName, "--clean", "--if-exists")
258245
} else {
259246
restoreCmd = append(restoreCmd, "-d", r.RestoreOptions.DBName)
260247
}

‎pkg/retrieval/engine/postgres/initialize/physical/physical.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
dblabCfg "gitlab.com/postgres-ai/database-lab/pkg/config"
2222
"gitlab.com/postgres-ai/database-lab/pkg/log"
2323
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
24+
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/dbmarker"
2425
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/engine/postgres/initialize/tools"
2526
"gitlab.com/postgres-ai/database-lab/pkg/retrieval/options"
2627
)
@@ -38,6 +39,7 @@ type RestoreJob struct {
3839
name string
3940
dockerClient *client.Client
4041
globalCfg *dblabCfg.Global
42+
dbMarker *dbmarker.Marker
4143
restorer restorer
4244
CopyOptions
4345
}
@@ -64,11 +66,12 @@ type restorer interface {
6466
}
6567

6668
// NewJob creates a new physical restore job.
67-
func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global) (*RestoreJob, error) {
69+
func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global, marker *dbmarker.Marker) (*RestoreJob, error) {
6870
physicalJob := &RestoreJob{
6971
name: cfg.Name,
7072
dockerClient: docker,
7173
globalCfg: global,
74+
dbMarker: marker,
7275
}
7376

7477
if err := options.Unmarshal(cfg.Options, &physicalJob.CopyOptions); err != nil {
@@ -181,6 +184,10 @@ func (r *RestoreJob) Run(ctx context.Context) error {
181184

182185
log.Msg("Restoring job has been finished")
183186

187+
if err := r.markDatabaseData(); err != nil {
188+
log.Err("Failed to mark database data: ", err)
189+
}
190+
184191
return nil
185192
}
186193

@@ -211,6 +218,14 @@ func (r *RestoreJob) getMountVolumes() []mount.Mount {
211218
return mounts
212219
}
213220

221+
func (r *RestoreJob) markDatabaseData() error {
222+
if err := r.dbMarker.CreateConfig(); err != nil {
223+
return errors.Wrap(err, "failed to create a DBMarker config of the database")
224+
}
225+
226+
return r.dbMarker.SaveConfig(&dbmarker.Config{DataType: dbmarker.PhysicalDataType})
227+
}
228+
214229
func waitForCommandResponse(ctx context.Context, attachResponse types.HijackedResponse) error {
215230
waitCommandCh := make(chan struct{})
216231

0 commit comments

Comments
 (0)