diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 52423a12c..bf9dd45e8 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -61,7 +61,10 @@ var ( backupHelperPath string restoreHelperPath string gprestorePath string - pluginConfigPath string + examplePluginDir string + examplePluginExec string + examplePluginTestConfig = "/tmp/test_example_plugin_config.yaml" + examplePluginTestDir = "/tmp/plugin_dest" // hardcoded in example plugin publicSchemaTupleCounts map[string]int schema2TupleCounts map[string]int backupDir string @@ -477,10 +480,27 @@ func TestEndToEnd(t *testing.T) { var _ = BeforeSuite(func() { // This is used to run tests from an older gpbackup version to gprestore latest useOldBackupVersion = os.Getenv("OLD_BACKUP_VERSION") != "" - pluginConfigPath = - fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin_config.yaml", - os.Getenv("GOPATH")) - var err error + + // Setup example plugin based on current working directory + err := os.RemoveAll(examplePluginTestDir) + Expect(err).ToNot(HaveOccurred()) + err = os.MkdirAll(examplePluginTestDir, 0777) + Expect(err).ToNot(HaveOccurred()) + currentDir, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + rootDir := path.Dir(currentDir) + examplePluginDir = path.Join(rootDir, "plugins") + examplePluginExec = path.Join(rootDir, "plugins", "example_plugin.bash") + examplePluginTestConfigContents := fmt.Sprintf(`executablepath: %s +options: + password: unknown`, examplePluginExec) + f, err := os.Create(examplePluginTestConfig) + Expect(err).ToNot(HaveOccurred()) + _, err = f.WriteString(examplePluginTestConfigContents) + Expect(err).ToNot(HaveOccurred()) + err = f.Close() + Expect(err).ToNot(HaveOccurred()) + testhelper.SetupTestLogger() _ = exec.Command("dropdb", "testdb").Run() _ = exec.Command("dropdb", "restoredb").Run() diff --git a/end_to_end/incremental_test.go b/end_to_end/incremental_test.go index d400f39a3..8c68f1fda 100644 --- a/end_to_end/incremental_test.go +++ b/end_to_end/incremental_test.go @@ -333,14 +333,13 @@ var _ = Describe("End to End incremental tests", func() { if useOldBackupVersion { Skip("This test is only needed for the most recent backup versions") } - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) }) It("Restores from an incremental backup based on a from-timestamp incremental", func() { fullBackupTimestamp := gpbackup(gpbackupPath, backupHelperPath, "--leaf-partition-data", "--single-data-file", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, fullBackupTimestamp) testhelper.AssertQueryRuns(backupConn, "INSERT into schema2.ao1 values(1001)") @@ -351,7 +350,7 @@ var _ = Describe("End to End incremental tests", func() { "--leaf-partition-data", "--single-data-file", "--from-timestamp", fullBackupTimestamp, - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, incremental1Timestamp) testhelper.AssertQueryRuns(backupConn, @@ -362,12 +361,12 @@ var _ = Describe("End to End incremental tests", func() { "--incremental", "--leaf-partition-data", "--single-data-file", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, incremental2Timestamp) gprestore(gprestorePath, restoreHelperPath, incremental2Timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -382,7 +381,7 @@ var _ = Describe("End to End incremental tests", func() { "--leaf-partition-data", "--single-data-file", "--copy-queue-size", "4", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, fullBackupTimestamp) testhelper.AssertQueryRuns(backupConn, "INSERT into schema2.ao1 values(1001)") @@ -394,7 +393,7 @@ var _ = Describe("End to End incremental tests", func() { "--single-data-file", "--copy-queue-size", "4", "--from-timestamp", fullBackupTimestamp, - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, incremental1Timestamp) testhelper.AssertQueryRuns(backupConn, @@ -406,13 +405,13 @@ var _ = Describe("End to End incremental tests", func() { "--leaf-partition-data", "--single-data-file", "--copy-queue-size", "4", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, incremental2Timestamp) gprestore(gprestorePath, restoreHelperPath, incremental2Timestamp, "--redirect-db", "restoredb", "--copy-queue-size", "4", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -423,13 +422,12 @@ var _ = Describe("End to End incremental tests", func() { assertArtifactsCleaned(restoreConn, incremental2Timestamp) }) It("Runs backup and restore if plugin location changed", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) fullBackupTimestamp := gpbackup(gpbackupPath, backupHelperPath, "--leaf-partition-data", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) otherPluginExecutablePath := fmt.Sprintf("%s/other_plugin_location/example_plugin.bash", backupDir) - command := exec.Command("bash", "-c", fmt.Sprintf("mkdir %s/other_plugin_location && cp %s %s/other_plugin_location", backupDir, pluginExecutablePath, backupDir)) + command := exec.Command("bash", "-c", fmt.Sprintf("mkdir %s/other_plugin_location && cp %s %s/other_plugin_location", backupDir, examplePluginExec, backupDir)) mustRunCommand(command) newCongig := fmt.Sprintf(`EOF1 executablepath: %s/other_plugin_location/example_plugin.bash diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index eedc8308a..8aedebe3c 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -20,8 +20,8 @@ func copyPluginToAllHosts(conn *dbconn.DBConn, pluginPath string) { hostnameQuery := `SELECT DISTINCT hostname AS string FROM gp_segment_configuration WHERE content != -1` hostnames := dbconn.MustSelectStringSlice(conn, hostnameQuery) for _, hostname := range hostnames { - pluginDir, _ := path.Split(pluginPath) - command := exec.Command("ssh", hostname, fmt.Sprintf("mkdir -p %s", pluginDir)) + examplePluginTestDir, _ := path.Split(pluginPath) + command := exec.Command("ssh", hostname, fmt.Sprintf("mkdir -p %s", examplePluginTestDir)) mustRunCommand(command) command = exec.Command("scp", pluginPath, fmt.Sprintf("%s:%s", hostname, pluginPath)) mustRunCommand(command) @@ -307,18 +307,17 @@ var _ = Describe("End to End plugin tests", func() { } }) It("runs gpbackup and gprestore with plugin, single-data-file, and no-compression", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--single-data-file", "--no-compression", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -326,19 +325,18 @@ var _ = Describe("End to End plugin tests", func() { assertArtifactsCleaned(restoreConn, timestamp) }) It("runs gpbackup and gprestore with plugin, single-data-file, no-compression, and copy-queue-size", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--single-data-file", "--copy-queue-size", "4", "--no-compression", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath, + "--plugin-config", examplePluginTestConfig, "--copy-queue-size", "4") assertRelationsCreated(restoreConn, TOTAL_RELATIONS) @@ -347,17 +345,16 @@ var _ = Describe("End to End plugin tests", func() { assertArtifactsCleaned(restoreConn, timestamp) }) It("runs gpbackup and gprestore with plugin and single-data-file", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--single-data-file", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -365,18 +362,17 @@ var _ = Describe("End to End plugin tests", func() { assertArtifactsCleaned(restoreConn, timestamp) }) It("runs gpbackup and gprestore with plugin, single-data-file, and copy-queue-size", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--single-data-file", "--copy-queue-size", "4", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath, + "--plugin-config", examplePluginTestConfig, "--copy-queue-size", "4") assertRelationsCreated(restoreConn, TOTAL_RELATIONS) @@ -385,17 +381,16 @@ var _ = Describe("End to End plugin tests", func() { assertArtifactsCleaned(restoreConn, timestamp) }) It("runs gpbackup and gprestore with plugin and metadata-only", func() { - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--metadata-only", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertArtifactsCleaned(restoreConn, timestamp) @@ -410,17 +405,16 @@ var _ = Describe("End to End plugin tests", func() { if useOldBackupVersion { Skip("This test is only needed for the most recent backup versions") } - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, "--no-compression", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -432,16 +426,15 @@ var _ = Describe("End to End plugin tests", func() { if useOldBackupVersion { Skip("This test is only needed for the most recent backup versions") } - pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, pluginExecutablePath) + copyPluginToAllHosts(backupConn, examplePluginExec) timestamp := gpbackup(gpbackupPath, backupHelperPath, - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) forceMetadataFileDownloadFromPlugin(backupConn, timestamp) gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", - "--plugin-config", pluginConfigPath) + "--plugin-config", examplePluginTestConfig) assertRelationsCreated(restoreConn, TOTAL_RELATIONS) assertDataRestored(restoreConn, publicSchemaTupleCounts) @@ -454,12 +447,9 @@ var _ = Describe("End to End plugin tests", func() { if useOldBackupVersion { Skip("This test is only needed for the latest backup version") } - pluginsDir := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins", os.Getenv("GOPATH")) - copyPluginToAllHosts(backupConn, fmt.Sprintf("%s/example_plugin.bash", pluginsDir)) - command := exec.Command("bash", "-c", fmt.Sprintf("%s/plugin_test.sh %s/example_plugin.bash %s/example_plugin_config.yaml /tmp/plugin_dest", pluginsDir, pluginsDir, pluginsDir)) + copyPluginToAllHosts(backupConn, examplePluginExec) + command := exec.Command("bash", "-c", fmt.Sprintf("%s/plugin_test.sh %s %s %s", examplePluginDir, examplePluginExec, examplePluginTestConfig, examplePluginTestDir)) mustRunCommand(command) - - _ = os.RemoveAll("/tmp/plugin_dest") }) }) })