Skip to content

Commit e9ec9af

Browse files
agneumagneum
committed
Merge branch '159-subdir-and-mountdir' into 'master'
refactor: allow having nested subdirectories for PGDATA relative to the pool mount directory(#159): - add data subdirectory - add mountpoint directory to the global config See merge request postgres-ai/database-lab!170
2 parents a0ff1ce + a8d3e98 commit e9ec9af

File tree

19 files changed

+144
-90
lines changed

19 files changed

+144
-90
lines changed

‎configs/config.example.logical_generic.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,18 @@ global:
2929
# Database engine. Currently, the only supported option: "postgres".
3030
engine: postgres
3131

32-
# Full path to data directory. This directory must already exist
33-
# before launching Database Lab instance. It may be empty if
32+
# Full path to the pool mount directory.
33+
mountDir: "/var/lib/dblab"
34+
35+
# Subdir where PGDATA located relative to the pool mount directory.
36+
# This directory must already exist before launching Database Lab instance. It may be empty if
3437
# data initialization is configured (see below).
35-
dataDir: "/var/lib/dblab/data"
38+
# Note, it is a relative path. Default: "data".
39+
# For example, for the PostgreSQL data directory "/var/lib/dblab/data" set:
40+
# mountDir: /var/lib/dblab
41+
# dataSubDir: data
42+
# In this case, we assume that the mount point is: /var/lib/dblab
43+
dataSubDir: "data"
3644

3745
# Debugging, when enabled, allows to see more in the Database Lab logs
3846
# (not PostgreSQL logs). Enable in the case of troubleshooting.

‎configs/config.example.logical_rds_iam.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,18 @@ global:
2929
# Database engine. Currently, the only supported option: "postgres".
3030
engine: postgres
3131

32-
# Full path to data directory. This directory must already exist
33-
# before launching Database Lab instance. It may be empty if
32+
# Full path to the pool mount directory.
33+
mountDir: "/var/lib/dblab"
34+
35+
# Subdir where PGDATA located relative to the pool mount directory.
36+
# This directory must already exist before launching Database Lab instance. It may be empty if
3437
# data initialization is configured (see below).
35-
dataDir: "/var/lib/dblab/data"
38+
# Note, it is a relative path. Default: "data".
39+
# For example, for the PostgreSQL data directory "/var/lib/dblab/data" set:
40+
# mountDir: /var/lib/dblab
41+
# dataSubDir: data
42+
# In this case, we assume that the mount point is: /var/lib/dblab
43+
dataSubDir: "data"
3644

3745
# Debugging, when enabled, allows to see more in the Database Lab logs
3846
# (not PostgreSQL logs). Enable in the case of troubleshooting.

‎configs/config.example.physical_generic.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,18 @@ global:
2929
# Database engine. Currently, the only supported option: "postgres".
3030
engine: postgres
3131

32-
# Full path to data directory. This directory must already exist
33-
# before launching Database Lab instance. It may be empty if
32+
# Full path to the pool mount directory.
33+
mountDir: "/var/lib/dblab"
34+
35+
# Subdir where PGDATA located relative to the pool mount directory.
36+
# This directory must already exist before launching Database Lab instance. It may be empty if
3437
# data initialization is configured (see below).
35-
dataDir: "/var/lib/dblab/data"
38+
# Note, it is a relative path. Default: "data".
39+
# For example, for the PostgreSQL data directory "/var/lib/dblab/data" set:
40+
# mountDir: /var/lib/dblab
41+
# dataSubDir: data
42+
# In this case, we assume that the mount point is: /var/lib/dblab
43+
dataSubDir: "data"
3644

3745
# Debugging, when enabled, allows to see more in the Database Lab logs
3846
# (not PostgreSQL logs). Enable in the case of troubleshooting.

‎configs/config.example.physical_walg.yml

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,18 @@ global:
2929
# Database engine. Currently, the only supported option: "postgres".
3030
engine: postgres
3131

32-
# Full path to data directory. This directory must already exist
33-
# before launching Database Lab instance. It may be empty if
32+
# Full path to the pool mount directory.
33+
mountDir: "/var/lib/dblab"
34+
35+
# Subdir where PGDATA located relative to the pool mount directory.
36+
# This directory must already exist before launching Database Lab instance. It may be empty if
3437
# data initialization is configured (see below).
35-
dataDir: "/var/lib/dblab/data"
38+
# Note, it is a relative path. Default: "data".
39+
# For example, for the PostgreSQL data directory "/var/lib/dblab/data" set:
40+
# mountDir: /var/lib/dblab
41+
# dataSubDir: data
42+
# In this case, we assume that the mount point is: /var/lib/dblab
43+
dataSubDir: "data"
3644

3745
# Debugging, when enabled, allows to see more in the Database Lab logs
3846
# (not PostgreSQL logs). Enable in the case of troubleshooting.

‎pkg/config/config.go

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,17 @@ import (
99
"fmt"
1010
"io/ioutil"
1111
"os/user"
12+
"path"
13+
14+
"github.com/pkg/errors"
15+
"gopkg.in/yaml.v2"
1216

1317
retConfig "gitlab.com/postgres-ai/database-lab/pkg/retrieval/config"
1418
"gitlab.com/postgres-ai/database-lab/pkg/services/cloning"
1519
"gitlab.com/postgres-ai/database-lab/pkg/services/platform"
1620
"gitlab.com/postgres-ai/database-lab/pkg/services/provision"
1721
"gitlab.com/postgres-ai/database-lab/pkg/srv"
1822
"gitlab.com/postgres-ai/database-lab/pkg/util"
19-
20-
"github.com/pkg/errors"
21-
"gopkg.in/yaml.v2"
2223
)
2324

2425
// Config contains a common database-lab configuration.
@@ -36,18 +37,24 @@ type Global struct {
3637
InstanceID string
3738
Engine string `yaml:"engine"`
3839
Debug bool `yaml:"debug"`
39-
DataDir string `yaml:"dataDir"`
40+
MountDir string `yaml:"mountDir"`
41+
DataSubDir string `yaml:"dataSubDir"`
4042
ClonesMountDir string // TODO (akartasov): Use ClonesMountDir for the LocalModeOptions of a Provision service.
4143
}
4244

45+
// DataDir provides full path to data directory.
46+
func (g Global) DataDir() string {
47+
return path.Join(g.MountDir, g.DataSubDir)
48+
}
49+
4350
// LoadConfig instances a new Config by configuration filename.
4451
func LoadConfig(name string) (*Config, error) {
45-
path, err := util.GetConfigPath(name)
52+
configPath, err := util.GetConfigPath(name)
4653
if err != nil {
4754
return nil, errors.Wrap(err, "failed to get config path")
4855
}
4956

50-
b, err := ioutil.ReadFile(path)
57+
b, err := ioutil.ReadFile(configPath)
5158
if err != nil {
5259
return nil, errors.Errorf("error loading %s config file", name)
5360
}
@@ -57,12 +64,22 @@ func LoadConfig(name string) (*Config, error) {
5764
return nil, fmt.Errorf("error parsing %s config", name)
5865
}
5966

67+
if err := cfg.setUpProvisionParams(); err != nil {
68+
return nil, errors.Wrap(err, "failed to set up provision options")
69+
}
70+
71+
return cfg, nil
72+
}
73+
74+
func (cfg *Config) setUpProvisionParams() error {
6075
osUser, err := user.Current()
6176
if err != nil {
62-
return nil, errors.Wrap(err, "failed to get current user")
77+
return errors.Wrap(err, "failed to get current user")
6378
}
6479

6580
cfg.Provision.OSUsername = osUser.Username
81+
cfg.Provision.MountDir = cfg.Global.MountDir
82+
cfg.Provision.DataSubDir = cfg.Global.DataSubDir
6683

67-
return cfg, nil
84+
return nil
6885
}

‎pkg/retrieval/engine/postgres/job_builder.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func NewJobBuilder(global *dblabCfg.Global, dockerClient *client.Client, cloneMa
3838
dockerClient: dockerClient,
3939
globalCfg: global,
4040
cloneManager: cloneManager,
41-
dbMarker: dbmarker.NewMarker(global.DataDir),
41+
dbMarker: dbmarker.NewMarker(global.DataDir()),
4242
}
4343
}
4444

‎pkg/retrieval/engine/postgres/logical/dump.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ func (d *DumpJob) Name() string {
198198
func (d *DumpJob) Run(ctx context.Context) (err error) {
199199
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", d.Name(), d.DumpOptions))
200200

201-
isEmpty, err := tools.IsEmptyDirectory(d.globalCfg.DataDir)
201+
isEmpty, err := tools.IsEmptyDirectory(d.globalCfg.DataDir())
202202
if err != nil {
203203
return errors.Wrap(err, "failed to explore the data directory")
204204
}
@@ -336,7 +336,7 @@ func (d *DumpJob) getEnvironmentVariables(password string) []string {
336336

337337
// Avoid initialization of PostgreSQL directory in case of preparing of a dump.
338338
if d.DumpOptions.Restore != nil {
339-
envs = append(envs, "PGDATA="+d.globalCfg.DataDir)
339+
envs = append(envs, "PGDATA="+d.globalCfg.DataDir())
340340
}
341341

342342
if d.DumpOptions.Source.Type == sourceTypeLocal && d.DumpOptions.Source.Connection.Port == defaults.Port {
@@ -361,7 +361,8 @@ func (d *DumpJob) buildHostConfig(ctx context.Context) (*container.HostConfig, e
361361
NetworkMode: d.getContainerNetworkMode(),
362362
}
363363

364-
if err := tools.AddVolumesToHostConfig(ctx, d.dockerClient, hostConfig, d.globalCfg.DataDir); err != nil {
364+
if err := tools.AddVolumesToHostConfig(ctx, d.dockerClient, hostConfig,
365+
d.globalCfg.MountDir, d.globalCfg.DataDir()); err != nil {
365366
return nil, err
366367
}
367368

‎pkg/retrieval/engine/postgres/logical/restore.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -102,18 +102,18 @@ func (r *RestoreJob) Name() string {
102102
func (r *RestoreJob) Run(ctx context.Context) (err error) {
103103
log.Msg(fmt.Sprintf("Run job: %s. Options: %v", r.Name(), r.RestoreOptions))
104104

105-
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir)
105+
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir())
106106
if err != nil {
107-
return errors.Wrapf(err, "failed to explore the data directory %q", r.globalCfg.DataDir)
107+
return errors.Wrapf(err, "failed to explore the data directory %q", r.globalCfg.DataDir())
108108
}
109109

110110
if !isEmpty {
111111
if !r.ForceInit {
112112
return errors.Errorf("the data directory %q is not empty. Use 'forceInit' or empty the data directory",
113-
r.globalCfg.DataDir)
113+
r.globalCfg.DataDir())
114114
}
115115

116-
log.Msg(fmt.Sprintf("The data directory %q is not empty. Existing data may be overwritten.", r.globalCfg.DataDir))
116+
log.Msg(fmt.Sprintf("The data directory %q is not empty. Existing data may be overwritten.", r.globalCfg.DataDir()))
117117
}
118118

119119
if err := tools.PullImage(ctx, r.dockerClient, r.RestoreOptions.DockerImage); err != nil {
@@ -204,7 +204,7 @@ func (r *RestoreJob) buildContainerConfig(password string) *container.Config {
204204
return &container.Config{
205205
Labels: map[string]string{tools.DBLabControlLabel: tools.DBLabRestoreLabel},
206206
Env: append(os.Environ(), []string{
207-
"PGDATA=" + r.globalCfg.DataDir,
207+
"PGDATA=" + r.globalCfg.DataDir(),
208208
"POSTGRES_PASSWORD=" + password,
209209
}...),
210210
Image: r.RestoreOptions.DockerImage,
@@ -215,7 +215,8 @@ func (r *RestoreJob) buildContainerConfig(password string) *container.Config {
215215
func (r *RestoreJob) buildHostConfig(ctx context.Context) (*container.HostConfig, error) {
216216
hostConfig := &container.HostConfig{}
217217

218-
if err := tools.AddVolumesToHostConfig(ctx, r.dockerClient, hostConfig, r.globalCfg.DataDir); err != nil {
218+
if err := tools.AddVolumesToHostConfig(ctx, r.dockerClient, hostConfig,
219+
r.globalCfg.MountDir, r.globalCfg.DataDir()); err != nil {
219220
return nil, err
220221
}
221222

‎pkg/retrieval/engine/postgres/physical/physical.go

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func NewJob(cfg config.JobConfig, docker *client.Client, global *dblabCfg.Global
100100
func (r *RestoreJob) getRestorer(tool string) (restorer, error) {
101101
switch tool {
102102
case walgTool:
103-
return newWalg(r.globalCfg.DataDir, r.WALG), nil
103+
return newWALG(r.globalCfg.DataDir(), r.WALG), nil
104104

105105
case customTool:
106106
return newCustomTool(r.CustomTool), nil
@@ -130,7 +130,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
130130
}
131131
}()
132132

133-
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir)
133+
isEmpty, err := tools.IsEmptyDirectory(r.globalCfg.DataDir())
134134
if err != nil {
135135
return errors.Wrap(err, "failed to explore the data directory")
136136
}
@@ -174,7 +174,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
174174
log.Err("Failed to mark database data: ", err)
175175
}
176176

177-
pgVersion, err := tools.DetectPGVersion(r.globalCfg.DataDir)
177+
pgVersion, err := tools.DetectPGVersion(r.globalCfg.DataDir())
178178
if err != nil {
179179
return errors.Wrap(err, "failed to detect the Postgres version")
180180
}
@@ -185,27 +185,27 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
185185
return errors.Wrap(err, "cannot get path to default configs")
186186
}
187187

188-
if err := fs.CopyDirectoryContent(sourceConfigDir, r.globalCfg.DataDir); err != nil {
188+
if err := fs.CopyDirectoryContent(sourceConfigDir, r.globalCfg.DataDir()); err != nil {
189189
return errors.Wrap(err, "failed to set default configuration files")
190190
}
191191

192-
if err := configuration.Run(r.globalCfg.DataDir); err != nil {
192+
if err := configuration.Run(r.globalCfg.DataDir()); err != nil {
193193
return errors.Wrap(err, "failed to configure")
194194
}
195195

196-
if err := r.adjustRecoveryConfiguration(pgVersion, r.globalCfg.DataDir); err != nil {
196+
if err := r.adjustRecoveryConfiguration(pgVersion, r.globalCfg.DataDir()); err != nil {
197197
return err
198198
}
199199

200200
// Set permissions.
201201
if err := tools.ExecCommand(ctx, r.dockerClient, contID, types.ExecConfig{
202-
Cmd: []string{"chown", "-R", "postgres", r.globalCfg.DataDir},
202+
Cmd: []string{"chown", "-R", "postgres", r.globalCfg.DataDir()},
203203
}); err != nil {
204204
return errors.Wrap(err, "failed to set permissions")
205205
}
206206

207207
// Start PostgreSQL instance.
208-
startCommand, err := r.dockerClient.ContainerExecCreate(ctx, contID, startingPostgresConfig(r.globalCfg.DataDir, pgVersion))
208+
startCommand, err := r.dockerClient.ContainerExecCreate(ctx, contID, startingPostgresConfig(r.globalCfg.DataDir(), pgVersion))
209209

210210
if err != nil {
211211
return errors.Wrap(err, "failed to create an exec command")
@@ -339,17 +339,17 @@ func (r *RestoreJob) runSyncInstance(ctx context.Context) error {
339339

340340
// Set permissions.
341341
if err := tools.ExecCommand(ctx, r.dockerClient, syncInstanceID, types.ExecConfig{
342-
Cmd: []string{"chown", "-R", "postgres", r.globalCfg.DataDir},
342+
Cmd: []string{"chown", "-R", "postgres", r.globalCfg.DataDir()},
343343
}); err != nil {
344344
return errors.Wrap(err, "failed to set permissions")
345345
}
346346

347-
pgVersion, err := tools.DetectPGVersion(r.globalCfg.DataDir)
347+
pgVersion, err := tools.DetectPGVersion(r.globalCfg.DataDir())
348348
if err != nil {
349349
return err
350350
}
351351

352-
startSyncCommand, err := r.dockerClient.ContainerExecCreate(ctx, syncInstanceID, startingPostgresConfig(r.globalCfg.DataDir, pgVersion))
352+
startSyncCommand, err := r.dockerClient.ContainerExecCreate(ctx, syncInstanceID, startingPostgresConfig(r.globalCfg.DataDir(), pgVersion))
353353
if err != nil {
354354
return errors.Wrap(err, "failed to create exec command")
355355
}
@@ -370,7 +370,7 @@ func (r *RestoreJob) getEnvironmentVariables(password string) []string {
370370
// Pass Database Lab environment variables.
371371
envVariables := append(os.Environ(), []string{
372372
"POSTGRES_PASSWORD=" + password,
373-
"PGDATA=" + r.globalCfg.DataDir,
373+
"PGDATA=" + r.globalCfg.DataDir(),
374374
}...)
375375

376376
// Add user-defined environment variables.
@@ -392,7 +392,8 @@ func (r *RestoreJob) buildContainerConfig(password, label string) *container.Con
392392
func (r *RestoreJob) buildHostConfig(ctx context.Context) (*container.HostConfig, error) {
393393
hostConfig := &container.HostConfig{}
394394

395-
if err := tools.AddVolumesToHostConfig(ctx, r.dockerClient, hostConfig, r.globalCfg.DataDir); err != nil {
395+
if err := tools.AddVolumesToHostConfig(ctx, r.dockerClient, hostConfig,
396+
r.globalCfg.MountDir, r.globalCfg.DataDir()); err != nil {
396397
return nil, err
397398
}
398399

‎pkg/retrieval/engine/postgres/physical/wal_g.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ type walgOptions struct {
2323
BackupName string `yaml:"backupName"`
2424
}
2525

26-
func newWalg(pgDataDir string, options walgOptions) *walg {
26+
func newWALG(pgDataDir string, options walgOptions) *walg {
2727
return &walg{
2828
pgDataDir: pgDataDir,
2929
options: options,

0 commit comments

Comments
 (0)