write only completed files to backup_content_control of running backup

alubennikova · alubennikova · commit 7ef294501a70 · 2019-06-11T12:58:50.000+03:00
diff --git a/src/backup.c b/src/backup.c
@@ -575,7 +575,7 @@ do_backup_instance(PGconn *backup_conn)
 
 	/* Print the list of files to backup catalog */
 	write_backup_filelist(&current, backup_files_list, instance_config.pgdata,
-						  NULL, external_dirs);
+						  external_dirs);
 
 	/* clean external directories list */
 	if (external_dirs)
@@ -1999,18 +1999,17 @@ backup_files(void *arg)
 		struct stat	buf;
 		pgFile	   *file = (pgFile *) parray_get(arguments->files_list, i);
 
-		
 		if (arguments->thread_num == 1)
 		{
 			/* update every 10 seconds */
 			if ((difftime(time(NULL), prev_time)) > 10)
 			{
 				prev_time = time(NULL);
-				elog(INFO, "write_backup_filelist N=%ld, starttime %ld, time %ld",
-					 parray_num(backup_files_list), current.start_time,  prev_time);
+				elog(INFO, "write_backup_filelist N=%d, starttime %ld, time %ld",
+					 n_backup_files_list, current.start_time,  prev_time);
 
-				write_backup_filelist(&current, backup_files_list, instance_config.pgdata,
-									NULL, arguments->external_dirs);
+				write_backup_filelist(&current, arguments->files_list, instance_config.pgdata,
+									  arguments->external_dirs);
 			}
 		}
 
@@ -2146,6 +2145,7 @@ backup_files(void *arg)
 				}
 			}
 
+			file->backuped = true;
 			elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",
 				 file->path, file->write_size);
 		}
diff --git a/src/catalog.c b/src/catalog.c
@@ -630,7 +630,7 @@ write_backup(pgBackup *backup)
  */
 void
 write_backup_filelist(pgBackup *backup, parray *files, const char *root,
-					  const char *external_prefix, parray *external_list)
+					  parray *external_list)
 {
 	FILE	   *out;
 	char		path[MAXPGPATH];
@@ -657,12 +657,15 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		char	line[BLCKSZ];
 		int 	len = 0;
 
+		i++;
+		if (!file->backuped)
+			continue;
+
 		/* omit root directory portion */
 		if (root && strstr(path, root) == path)
 			path = GetRelativePath(path, root);
-		else if (file->external_dir_num && !external_prefix)
+		else if (file->external_dir_num && external_list)
 		{
-			Assert(external_list);
 			path = GetRelativePath(path, parray_get(external_list,
 													file->external_dir_num - 1));
 		}
@@ -707,8 +710,6 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 			/* reset write_len */
 			write_len = 0;
 		}
-
-		i++;
 	}
 
 	/* write what is left in the buffer to file */
diff --git a/src/merge.c b/src/merge.c
@@ -368,8 +368,7 @@ merge_backups(pgBackup *to_backup, pgBackup *from_backup)
 	else
 		to_backup->wal_bytes = BYTES_INVALID;
 
-	write_backup_filelist(to_backup, files, from_database_path,
-						  from_external_prefix, NULL);
+	write_backup_filelist(to_backup, files, from_database_path, NULL);
 	write_backup(to_backup);
 
 delete_source_backup:
diff --git a/src/pg_probackup.h b/src/pg_probackup.h
@@ -146,6 +146,9 @@ typedef struct pgFile
 	datapagemap_t pagemap;	/* bitmap of pages updated since previous backup */
 	bool	pagemap_isabsent; /* Used to mark files with unknown state of pagemap,
 							   * i.e. datafiles without _ptrack */
+
+	/* state during bakup */
+	bool backuped; /* is file already completely copied into destination backup? */
 } pgFile;
 
 /* Special values of datapagemap_t bitmapsize */
@@ -555,8 +558,7 @@ extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
 											  TimeLineID tli);
 extern void pgBackupWriteControl(FILE *out, pgBackup *backup);
 extern void write_backup_filelist(pgBackup *backup, parray *files,
-								  const char *root, const char *external_prefix,
-								  parray *external_list);
+								  const char *root, parray *external_list);
 
 extern void pgBackupGetPath(const pgBackup *backup, char *path, size_t len,
 							const char *subdir);

Original file line number	Diff line number	Diff line change
`@@ -575,7 +575,7 @@ do_backup_instance(PGconn *backup_conn)`
`575`	`575`
`576`	`576`	`/* Print the list of files to backup catalog */`
`577`	`577`	`write_backup_filelist(&current, backup_files_list, instance_config.pgdata,`
`578`		`- NULL, external_dirs);`
	`578`	`+ external_dirs);`
`579`	`579`
`580`	`580`	`/* clean external directories list */`
`581`	`581`	`if (external_dirs)`
`@@ -1999,18 +1999,17 @@ backup_files(void *arg)`
`1999`	`1999`	`struct stat buf;`
`2000`	`2000`	`pgFile file = (pgFile ) parray_get(arguments->files_list, i);`
`2001`	`2001`
`2002`		`-`
`2003`	`2002`	`if (arguments->thread_num == 1)`
`2004`	`2003`	`{`
`2005`	`2004`	`/* update every 10 seconds */`
`2006`	`2005`	`if ((difftime(time(NULL), prev_time)) > 10)`
`2007`	`2006`	`{`
`2008`	`2007`	`prev_time = time(NULL);`
`2009`		`- elog(INFO, "write_backup_filelist N=%ld, starttime %ld, time %ld",`
`2010`		`- parray_num(backup_files_list), current.start_time, prev_time);`
	`2008`	`+ elog(INFO, "write_backup_filelist N=%d, starttime %ld, time %ld",`
	`2009`	`+ n_backup_files_list, current.start_time, prev_time);`
`2011`	`2010`
`2012`		`- write_backup_filelist(&current, backup_files_list, instance_config.pgdata,`
`2013`		`- NULL, arguments->external_dirs);`
	`2011`	`+ write_backup_filelist(&current, arguments->files_list, instance_config.pgdata,`
	`2012`	`+ arguments->external_dirs);`
`2014`	`2013`	`}`
`2015`	`2014`	`}`
`2016`	`2015`
`@@ -2146,6 +2145,7 @@ backup_files(void *arg)`
`2146`	`2145`	`}`
`2147`	`2146`	`}`
`2148`	`2147`
	`2148`	`+ file->backuped = true;`
`2149`	`2149`	`elog(VERBOSE, "File \"%s\". Copied "INT64_FORMAT " bytes",`
`2150`	`2150`	`file->path, file->write_size);`
`2151`	`2151`	`}`