Skip to content

Commit

Permalink
End_to_end: fix example plugin end_to_end test to use relative paths
Browse files Browse the repository at this point in the history 
When gpbackup repo is not in $GOPATH and end_to_end tests are run, tests
that use the example plugin will hang because the test cannot proceed
without the example plugin reading data from test pipes. This happens
because the example plugin and it's config file locations were hardcoded
to be in $GOPATH. This is a legacy issue from when go projects were
required to be in $GOPATH. This requirement was changed to be optional
in go1.11. This is an update to look for example plugin using a relative
path. The test plugin config file is also now setup before tests run
because the executablepath is also updated to be a relative path.
  • Loading branch information
@kyeap-vmware
kyeap-vmware committed Feb 2, 2024
1 parent 0c22043 commit 92245fe
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 53 deletions.
30 changes: 25 additions & 5 deletions end_to_end/end_to_end_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,10 @@ var (
backupHelperPath string
restoreHelperPath string
gprestorePath string
pluginConfigPath string
examplePluginDir string
examplePluginExec string
examplePluginTestConfig = "/tmp/test_example_plugin_config.yaml"
examplePluginTestDir = "/tmp/plugin_dest" // hardcoded in example plugin
publicSchemaTupleCounts map[string]int
schema2TupleCounts map[string]int
backupDir string
Expand Down Expand Up @@ -477,10 +480,27 @@ func TestEndToEnd(t *testing.T) {
var _ = BeforeSuite(func() {
// This is used to run tests from an older gpbackup version to gprestore latest
useOldBackupVersion = os.Getenv("OLD_BACKUP_VERSION") != ""
pluginConfigPath =
fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin_config.yaml",
os.Getenv("GOPATH"))
var err error

// Setup example plugin based on current working directory
err := os.RemoveAll(examplePluginTestDir)
Expect(err).ToNot(HaveOccurred())
err = os.MkdirAll(examplePluginTestDir, 0777)
Expect(err).ToNot(HaveOccurred())
currentDir, err := os.Getwd()
Expect(err).ToNot(HaveOccurred())
rootDir := path.Dir(currentDir)
examplePluginDir = path.Join(rootDir, "plugins")
examplePluginExec = path.Join(rootDir, "plugins", "example_plugin.bash")
examplePluginTestConfigContents := fmt.Sprintf(`executablepath: %s
options:
password: unknown`, examplePluginExec)
f, err := os.Create(examplePluginTestConfig)
Expect(err).ToNot(HaveOccurred())
_, err = f.WriteString(examplePluginTestConfigContents)
Expect(err).ToNot(HaveOccurred())
err = f.Close()
Expect(err).ToNot(HaveOccurred())

testhelper.SetupTestLogger()
_ = exec.Command("dropdb", "testdb").Run()
_ = exec.Command("dropdb", "restoredb").Run()
Expand Down
24 changes: 11 additions & 13 deletions end_to_end/incremental_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,14 +333,13 @@ var _ = Describe("End to End incremental tests", func() {
if useOldBackupVersion {
Skip("This test is only needed for the most recent backup versions")
}
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)
})
It("Restores from an incremental backup based on a from-timestamp incremental", func() {
fullBackupTimestamp := gpbackup(gpbackupPath, backupHelperPath,
"--leaf-partition-data",
"--single-data-file",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, fullBackupTimestamp)
testhelper.AssertQueryRuns(backupConn,
"INSERT into schema2.ao1 values(1001)")
Expand All @@ -351,7 +350,7 @@ var _ = Describe("End to End incremental tests", func() {
"--leaf-partition-data",
"--single-data-file",
"--from-timestamp", fullBackupTimestamp,
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, incremental1Timestamp)

testhelper.AssertQueryRuns(backupConn,
Expand All @@ -362,12 +361,12 @@ var _ = Describe("End to End incremental tests", func() {
"--incremental",
"--leaf-partition-data",
"--single-data-file",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, incremental2Timestamp)

gprestore(gprestorePath, restoreHelperPath, incremental2Timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
Expand All @@ -382,7 +381,7 @@ var _ = Describe("End to End incremental tests", func() {
"--leaf-partition-data",
"--single-data-file",
"--copy-queue-size", "4",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, fullBackupTimestamp)
testhelper.AssertQueryRuns(backupConn,
"INSERT into schema2.ao1 values(1001)")
Expand All @@ -394,7 +393,7 @@ var _ = Describe("End to End incremental tests", func() {
"--single-data-file",
"--copy-queue-size", "4",
"--from-timestamp", fullBackupTimestamp,
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, incremental1Timestamp)

testhelper.AssertQueryRuns(backupConn,
Expand All @@ -406,13 +405,13 @@ var _ = Describe("End to End incremental tests", func() {
"--leaf-partition-data",
"--single-data-file",
"--copy-queue-size", "4",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, incremental2Timestamp)

gprestore(gprestorePath, restoreHelperPath, incremental2Timestamp,
"--redirect-db", "restoredb",
"--copy-queue-size", "4",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
Expand All @@ -423,13 +422,12 @@ var _ = Describe("End to End incremental tests", func() {
assertArtifactsCleaned(restoreConn, incremental2Timestamp)
})
It("Runs backup and restore if plugin location changed", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
fullBackupTimestamp := gpbackup(gpbackupPath, backupHelperPath,
"--leaf-partition-data",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

otherPluginExecutablePath := fmt.Sprintf("%s/other_plugin_location/example_plugin.bash", backupDir)
command := exec.Command("bash", "-c", fmt.Sprintf("mkdir %s/other_plugin_location && cp %s %s/other_plugin_location", backupDir, pluginExecutablePath, backupDir))
command := exec.Command("bash", "-c", fmt.Sprintf("mkdir %s/other_plugin_location && cp %s %s/other_plugin_location", backupDir, examplePluginExec, backupDir))
mustRunCommand(command)
newCongig := fmt.Sprintf(`EOF1
executablepath: %s/other_plugin_location/example_plugin.bash
Expand Down
60 changes: 25 additions & 35 deletions end_to_end/plugin_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ func copyPluginToAllHosts(conn *dbconn.DBConn, pluginPath string) {
hostnameQuery := `SELECT DISTINCT hostname AS string FROM gp_segment_configuration WHERE content != -1`
hostnames := dbconn.MustSelectStringSlice(conn, hostnameQuery)
for _, hostname := range hostnames {
pluginDir, _ := path.Split(pluginPath)
command := exec.Command("ssh", hostname, fmt.Sprintf("mkdir -p %s", pluginDir))
examplePluginTestDir, _ := path.Split(pluginPath)
command := exec.Command("ssh", hostname, fmt.Sprintf("mkdir -p %s", examplePluginTestDir))
mustRunCommand(command)
command = exec.Command("scp", pluginPath, fmt.Sprintf("%s:%s", hostname, pluginPath))
mustRunCommand(command)
Expand Down Expand Up @@ -307,38 +307,36 @@ var _ = Describe("End to End plugin tests", func() {
}
})
It("runs gpbackup and gprestore with plugin, single-data-file, and no-compression", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--single-data-file",
"--no-compression",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
assertDataRestored(restoreConn, schema2TupleCounts)
assertArtifactsCleaned(restoreConn, timestamp)
})
It("runs gpbackup and gprestore with plugin, single-data-file, no-compression, and copy-queue-size", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--single-data-file",
"--copy-queue-size", "4",
"--no-compression",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath,
"--plugin-config", examplePluginTestConfig,
"--copy-queue-size", "4")

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
Expand All @@ -347,36 +345,34 @@ var _ = Describe("End to End plugin tests", func() {
assertArtifactsCleaned(restoreConn, timestamp)
})
It("runs gpbackup and gprestore with plugin and single-data-file", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--single-data-file",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
assertDataRestored(restoreConn, schema2TupleCounts)
assertArtifactsCleaned(restoreConn, timestamp)
})
It("runs gpbackup and gprestore with plugin, single-data-file, and copy-queue-size", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--single-data-file",
"--copy-queue-size", "4",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath,
"--plugin-config", examplePluginTestConfig,
"--copy-queue-size", "4")

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
Expand All @@ -385,17 +381,16 @@ var _ = Describe("End to End plugin tests", func() {
assertArtifactsCleaned(restoreConn, timestamp)
})
It("runs gpbackup and gprestore with plugin and metadata-only", func() {
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--metadata-only",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertArtifactsCleaned(restoreConn, timestamp)
Expand All @@ -410,17 +405,16 @@ var _ = Describe("End to End plugin tests", func() {
if useOldBackupVersion {
Skip("This test is only needed for the most recent backup versions")
}
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--no-compression",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
Expand All @@ -432,16 +426,15 @@ var _ = Describe("End to End plugin tests", func() {
if useOldBackupVersion {
Skip("This test is only needed for the most recent backup versions")
}
pluginExecutablePath := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins/example_plugin.bash", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, pluginExecutablePath)
copyPluginToAllHosts(backupConn, examplePluginExec)

timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)
forceMetadataFileDownloadFromPlugin(backupConn, timestamp)

gprestore(gprestorePath, restoreHelperPath, timestamp,
"--redirect-db", "restoredb",
"--plugin-config", pluginConfigPath)
"--plugin-config", examplePluginTestConfig)

assertRelationsCreated(restoreConn, TOTAL_RELATIONS)
assertDataRestored(restoreConn, publicSchemaTupleCounts)
Expand All @@ -454,12 +447,9 @@ var _ = Describe("End to End plugin tests", func() {
if useOldBackupVersion {
Skip("This test is only needed for the latest backup version")
}
pluginsDir := fmt.Sprintf("%s/src/github.com/greenplum-db/gpbackup/plugins", os.Getenv("GOPATH"))
copyPluginToAllHosts(backupConn, fmt.Sprintf("%s/example_plugin.bash", pluginsDir))
command := exec.Command("bash", "-c", fmt.Sprintf("%s/plugin_test.sh %s/example_plugin.bash %s/example_plugin_config.yaml /tmp/plugin_dest", pluginsDir, pluginsDir, pluginsDir))
copyPluginToAllHosts(backupConn, examplePluginExec)
command := exec.Command("bash", "-c", fmt.Sprintf("%s/plugin_test.sh %s %s %s", examplePluginDir, examplePluginExec, examplePluginTestConfig, examplePluginTestDir))
mustRunCommand(command)

_ = os.RemoveAll("/tmp/plugin_dest")
})
})
})

0 comments on commit 92245fe

Please sign in to comment.