Skip to content

Commit

Permalink
Prevent partition queries from running in gpdb7
Browse files Browse the repository at this point in the history 
In GPDB 7+, partition tables have been heavily revamped to match the
upstream Postgres implementation. Due to this, many existing gpbackup
partitioning logic will simply error out due to missing catalog
functions, catalog views, and catalog tables. As a first step in
preparation for upgrading the gpbackup to support gpdb7, prevent
any queries that use the following deprecated code.

Catalog functions removed:
pg_get_partition_def
pg_get_partition_template_def

Catalog tables removed:
pg_partitions
pg_partition_columns
pg_partition_templates
pg_partition
pg_partition_rule
pg_partition_encoding

Authored-by: Kevin Yeap <kyeap@vmware.com>
  • Loading branch information
@kyeap-vmware @AJR-VMware
kyeap-vmware authored and AJR-VMware committed Nov 22, 2022
1 parent 0c7ec12 commit ea1c002
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 52 deletions.
5 changes: 5 additions & 0 deletions backup/queries_externals.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,11 @@ func (pi PartitionInfo) GetMetadataEntry() (string, toc.MetadataEntry) {
}

func GetExternalPartitionInfo(connectionPool *dbconn.DBConn) ([]PartitionInfo, map[uint32]PartitionInfo) {
// TODO: fix for gpdb7 partitioning
if connectionPool.Version.AtLeast("7") {
return []PartitionInfo{}, make(map[uint32]PartitionInfo, 0)
}

results := make([]PartitionInfo, 0)
query := `
SELECT pr1.oid AS partitionruleoid,
Expand Down
30 changes: 18 additions & 12 deletions backup/queries_postdata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,15 @@ func (i IndexDefinition) FQN() string {
* e.g. comments on implicitly created indexes
*/
func GetIndexes(connectionPool *dbconn.DBConn) []IndexDefinition {
resultIndexes := make([]IndexDefinition, 0)

var query string
if connectionPool.Version.Before("6") {
indexOidList := ConstructImplicitIndexOidList(connectionPool)
implicitIndexStr := ""
if indexOidList != "" {
implicitIndexStr = fmt.Sprintf("OR i.indexrelid IN (%s)", indexOidList)
}
query := fmt.Sprintf(`
query = fmt.Sprintf(`
SELECT DISTINCT i.indexrelid AS oid,
quote_ident(ic.relname) AS name,
quote_ident(n.nspname) AS owningschema,
Expand All @@ -108,12 +109,16 @@ func GetIndexes(connectionPool *dbconn.DBConn) []IndexDefinition {
AND NOT EXISTS (SELECT 1 FROM pg_partition_rule r WHERE r.parchildrelid = c.oid)
AND %s
ORDER BY name`,
implicitIndexStr, relationAndSchemaFilterClause(), ExtensionFilterClause("c"))
implicitIndexStr, relationAndSchemaFilterClause(), ExtensionFilterClause("c"))

err := connectionPool.Select(&resultIndexes, query)
gplog.FatalOnError(err)
} else {
query := fmt.Sprintf(`
// TODO: fix for gpdb7 partitioning
partitionRuleExcludeClause := ""
if connectionPool.Version.Before("7") {
partitionRuleExcludeClause = "AND NOT EXISTS (SELECT 1 FROM pg_partition_rule r WHERE r.parchildrelid = c.oid)"
}

query = fmt.Sprintf(`
SELECT DISTINCT i.indexrelid AS oid,
quote_ident(ic.relname) AS name,
quote_ident(n.nspname) AS owningschema,
Expand All @@ -138,14 +143,15 @@ func GetIndexes(connectionPool *dbconn.DBConn) []IndexDefinition {
AND i.indisvalid
AND i.indisready
AND i.indisprimary = 'f'
AND NOT EXISTS (SELECT 1 FROM pg_partition_rule r WHERE r.parchildrelid = c.oid)
%s
AND %s
ORDER BY name`,
relationAndSchemaFilterClause(), ExtensionFilterClause("c")) // The index itself does not have a dependency on the extension, but the index's table does
err := connectionPool.Select(&resultIndexes, query)
gplog.FatalOnError(err)
relationAndSchemaFilterClause(), partitionRuleExcludeClause, ExtensionFilterClause("c")) // The index itself does not have a dependency on the extension, but the index's table does
}

resultIndexes := make([]IndexDefinition, 0)
err := connectionPool.Select(&resultIndexes, query)
gplog.FatalOnError(err)
// Remove all indexes that have NULL definitions. This can happen
// if a concurrent index drop happens before the associated table
// lock is acquired earlier during gpbackup execution.
Expand Down Expand Up @@ -211,7 +217,7 @@ func GetRules(connectionPool *dbconn.DBConn) []RuleDefinition {
AND rulename NOT LIKE 'pg_%%'
AND %s
ORDER BY rulename`,
relationAndSchemaFilterClause(), ExtensionFilterClause("c"))
relationAndSchemaFilterClause(), ExtensionFilterClause("c"))

results := make([]RuleDefinition, 0)
err := connectionPool.Select(&results, query)
Expand Down Expand Up @@ -275,7 +281,7 @@ func GetTriggers(connectionPool *dbconn.DBConn) []TriggerDefinition {
AND %s
AND %s
ORDER BY tgname`,
relationAndSchemaFilterClause(), constraintClause, ExtensionFilterClause("c"))
relationAndSchemaFilterClause(), constraintClause, ExtensionFilterClause("c"))

results := make([]TriggerDefinition, 0)
err := connectionPool.Select(&results, query)
Expand Down
3 changes: 2 additions & 1 deletion backup/queries_relations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,9 @@ func (r Relation) GetUniqueID() UniqueID {
* it is currently much simpler than the include case.
*/
func getUserTableRelations(connectionPool *dbconn.DBConn) []Relation {
// TODO: fix for gpdb7 partitioning
childPartitionFilter := ""
if !MustGetFlagBool(options.LEAF_PARTITION_DATA) {
if !MustGetFlagBool(options.LEAF_PARTITION_DATA) && connectionPool.Version.Before("7") {
//Filter out non-external child partitions
childPartitionFilter = `
AND c.oid NOT IN (
Expand Down
12 changes: 11 additions & 1 deletion backup/queries_shared.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,17 @@ func (c Constraint) FQN() string {
}

func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) []Constraint {
// ConIsLocal should always return true from GetConstraints because we filter out constraints that are inherited using the INHERITS clause, or inherited from a parent partition table. This field only accurately reflects constraints in GPDB6+ because check constraints on parent tables must propogate to children. For GPDB versions 5 or lower, this field will default to false.
// TODO: fix for gpdb7 partitioning
if connectionPool.Version.AtLeast("7") {
return []Constraint{}
}

// ConIsLocal should always return true from GetConstraints because we
// filter out constraints that are inherited using the INHERITS clause, or
// inherited from a parent partition table. This field only accurately
// reflects constraints in GPDB6+ because check constraints on parent
// tables must propogate to children. For GPDB versions 5 or lower, this
// field will default to false.
var selectConIsLocal string
var groupByConIsLocal string
if connectionPool.Version.AtLeast("6") {
Expand Down
97 changes: 59 additions & 38 deletions backup/queries_table_defs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,20 +57,20 @@ func createAlteredPartitionSchemaSet(tables []Table) map[string]bool {
}

type TableDefinition struct {
DistPolicy string
PartDef string
PartTemplateDef string
StorageOpts string
TablespaceName string
ColumnDefs []ColumnDefinition
IsExternal bool
ExtTableDef ExternalTableDefinition
PartitionLevelInfo PartitionLevelInfo
TableType string
IsUnlogged bool
ForeignDef ForeignTableDefinition
Inherits []string
ReplicaIdentity string
DistPolicy string
PartDef string
PartTemplateDef string
StorageOpts string
TablespaceName string
ColumnDefs []ColumnDefinition
IsExternal bool
ExtTableDef ExternalTableDefinition
PartitionLevelInfo PartitionLevelInfo
TableType string
IsUnlogged bool
ForeignDef ForeignTableDefinition
Inherits []string
ReplicaIdentity string
PartitionAlteredSchemas []AlteredPartitionRelation
AccessMethodName string
}
Expand Down Expand Up @@ -102,20 +102,20 @@ func ConstructDefinitionsForTables(connectionPool *dbconn.DBConn, tableRelations
for _, tableRel := range tableRelations {
oid := tableRel.Oid
tableDef := TableDefinition{
DistPolicy: distributionPolicies[oid],
PartDef: partitionDefs[oid],
PartTemplateDef: partTemplateDefs[oid],
StorageOpts: storageOptions[oid],
TablespaceName: tablespaceNames[oid],
ColumnDefs: columnDefs[oid],
IsExternal: extTableDefs[oid].Oid != 0,
ExtTableDef: extTableDefs[oid],
PartitionLevelInfo: partTableMap[oid],
TableType: tableTypeMap[oid],
IsUnlogged: unloggedTableMap[oid],
ForeignDef: foreignTableDefs[oid],
Inherits: inheritanceMap[oid],
ReplicaIdentity: replicaIdentityMap[oid],
DistPolicy: distributionPolicies[oid],
PartDef: partitionDefs[oid],
PartTemplateDef: partTemplateDefs[oid],
StorageOpts: storageOptions[oid],
TablespaceName: tablespaceNames[oid],
ColumnDefs: columnDefs[oid],
IsExternal: extTableDefs[oid].Oid != 0,
ExtTableDef: extTableDefs[oid],
PartitionLevelInfo: partTableMap[oid],
TableType: tableTypeMap[oid],
IsUnlogged: unloggedTableMap[oid],
ForeignDef: foreignTableDefs[oid],
Inherits: inheritanceMap[oid],
ReplicaIdentity: replicaIdentityMap[oid],
PartitionAlteredSchemas: partitionAlteredSchemaMap[oid],
AccessMethodName: accessMethodMap[oid],
}
Expand All @@ -140,6 +140,11 @@ type PartitionLevelInfo struct {
}

func GetPartitionTableMap(connectionPool *dbconn.DBConn) map[uint32]PartitionLevelInfo {
// TODO: fix for gpdb7 partitioning
if connectionPool.Version.AtLeast("7") {
return make(map[uint32]PartitionLevelInfo)
}

query := `
SELECT pc.oid AS oid,
'p' AS level,
Expand Down Expand Up @@ -228,15 +233,22 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi
LEFT JOIN pg_catalog.pg_type t ON a.atttypid = t.oid
LEFT JOIN pg_catalog.pg_attribute_encoding e ON e.attrelid = a.attrelid AND e.attnum = a.attnum
LEFT JOIN pg_description d ON d.objoid = a.attrelid AND d.classoid = 'pg_class'::regclass AND d.objsubid = a.attnum`
whereClause := `
WHERE ` + relationAndSchemaFilterClause() + `

// TODO: fix for gpdb7 partitioning
partitionRuleExcludeClause := ""
if connectionPool.Version.Before("7") {
partitionRuleExcludeClause = `
AND NOT EXISTS (SELECT 1 FROM
(SELECT parchildrelid FROM pg_partition_rule EXCEPT SELECT reloid FROM pg_exttable)
par WHERE par.parchildrelid = c.oid)
par WHERE par.parchildrelid = c.oid)`
}
whereClause := fmt.Sprintf(`
WHERE %s
%s
AND c.reltype <> 0
AND a.attnum > 0::pg_catalog.int2
AND a.attisdropped = 'f'
ORDER BY a.attrelid, a.attnum`
ORDER BY a.attrelid, a.attnum`, relationAndSchemaFilterClause(), partitionRuleExcludeClause)

if connectionPool.Version.AtLeast("6") {
// Cannot use unnest() in CASE statements anymore in GPDB 7+ so convert
Expand Down Expand Up @@ -339,7 +351,7 @@ func GetTableAccessMethod(connectionPool *dbconn.DBConn) map[uint32]string {

func GetTableReplicaIdentity(connectionPool *dbconn.DBConn) map[uint32]string {
if connectionPool.Version.Before("6") {
return map[uint32]string{}
return make(map[uint32]string)
}
query := fmt.Sprintf(`
SELECT oid,
Expand All @@ -351,6 +363,10 @@ func GetTableReplicaIdentity(connectionPool *dbconn.DBConn) map[uint32]string {
}

func GetPartitionDetails(connectionPool *dbconn.DBConn) (map[uint32]string, map[uint32]string) {
if connectionPool.Version.AtLeast("7") {
return make(map[uint32]string), make(map[uint32]string)
}

gplog.Info("Getting partition definitions")
query := fmt.Sprintf(`
SELECT p.parrelid AS oid,
Expand Down Expand Up @@ -379,9 +395,9 @@ func GetPartitionDetails(connectionPool *dbconn.DBConn) (map[uint32]string, map[
}

type AlteredPartitionRelation struct {
OldSchema string
NewSchema string
Name string
OldSchema string
NewSchema string
Name string
}

/*
Expand All @@ -391,6 +407,10 @@ type AlteredPartitionRelation struct {
* them.
*/
func GetPartitionAlteredSchema(connectionPool *dbconn.DBConn) map[uint32][]AlteredPartitionRelation {
if connectionPool.Version.AtLeast("7") {
return make(map[uint32][]AlteredPartitionRelation)
}

gplog.Info("Getting child partitions with altered schema")
query := fmt.Sprintf(`
SELECT pgp.parrelid AS oid,
Expand All @@ -405,7 +425,7 @@ func GetPartitionAlteredSchema(connectionPool *dbconn.DBConn) map[uint32][]Alter
JOIN pg_catalog.pg_namespace pgn2 ON pgc2.relnamespace = pgn2.oid
WHERE pgc.relnamespace != pgc2.relnamespace`)
var results []struct {
Oid uint32
Oid uint32
AlteredPartitionRelation
}
err := connectionPool.Select(&results, query)
Expand Down Expand Up @@ -477,6 +497,7 @@ func GetForeignTableDefinitions(connectionPool *dbconn.DBConn) map[uint32]Foreig
if connectionPool.Version.Before("6") {
return map[uint32]ForeignTableDefinition{}
}

query := fmt.Sprintf(`
SELECT ftrelid, fs.srvname AS ftserver,
pg_catalog.array_to_string(array(
Expand Down Expand Up @@ -522,7 +543,7 @@ func GetTableInheritance(connectionPool *dbconn.DBConn, tables []Relation) map[u
JOIN pg_namespace n ON p.relnamespace = n.oid
WHERE %s%s
ORDER BY i.inhrelid, i.inhseqno`,
ExtensionFilterClause("p"), tableFilterStr)
ExtensionFilterClause("p"), tableFilterStr)

results := make([]Dependency, 0)
resultMap := make(map[uint32][]string)
Expand Down
5 changes: 5 additions & 0 deletions options/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,11 @@ func (o *Options) QuoteExcludeRelations(conn *dbconn.DBConn) error {
}

func (o Options) getUserTableRelationsWithIncludeFiltering(connectionPool *dbconn.DBConn, includedRelationsQuoted []string) ([]FqnStruct, error) {
// TODO: fix for gpdb7 partitioning
if connectionPool.Version.AtLeast("7") {
return []FqnStruct{}, nil
}

includeOids, err := getOidsFromRelationList(connectionPool, includedRelationsQuoted)
if err != nil {
return nil, err
Expand Down

0 comments on commit ea1c002

Please sign in to comment.