Improved Purview Integration (Azure#190)

* Improved Purview Integration * updated input parameters to reduce # of req RBAC * updated portal ui to include managed resource ids as input * added docs * made settings consistent * updated parameter files * upgrade bicep version
rhadi2005 · Oct 25, 2021 · 94f89cb · 94f89cb
1 parent b286f2e
commit 94f89cb
Show file tree

Hide file tree

Showing 10 changed files with 218 additions and 32 deletions.
diff --git a/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md b/docs/EnterpriseScaleAnalytics-AzureDevOpsDeployment.md
@@ -88,6 +88,8 @@ To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In th
 | dnsServerAdresses | Specifies the private IP addresses of the dns servers. | `[ 10.0.0.4 ]` |
 | administratorPassword | Specifies the administrator password of the sql servers. Will be automatically set in the workflow. **Leave this value as is.** | `<your-secure-password>` |
 | purviewId | Specifies the resource ID of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` |
+| purviewManagedStorageId | Specifies the resource ID of the managed storage of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-account-name}` |
+| purviewManagedEventHubId | Specifies the resource ID of the managed event hub of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.EventHub/namespaces/{eventhub-namespace-name}` |
 | purviewSelfHostedIntegrationRuntimeAuthKey | Specifies the Auth Key for the Self-hosted integration runtime of Purview. | `<your-purview-shir-auth-key>` |
 | deploySelfHostedIntegrationRuntimes | Specifies whether the self-hosted integration runtimes should be installed. This only works, if the pwsh script was uploded and is available. | `true` or `false` |
 | privateDnsZoneIdKeyVault | Specifies the resource ID of the private DNS zone for KeyVault. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Network/privateDnsZones/privatelink.vaultcore.azure.net` |

diff --git a/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md b/docs/EnterpriseScaleAnalytics-GitHubActionsDeployment.md
@@ -80,6 +80,8 @@ To begin, please open the [infra/params.dev.json](/infra/params.dev.json). In th
 | dnsServerAdresses | Specifies the private IP addresses of the DNS Servers. | `[ 10.0.0.4 ]` |
 | administratorPassword | Specifies the administrator password of the SQL Servers. Will be automatically set in the workflow. **Leave this value as is.** | `<your-secure-password>` |
 | purviewId | Specifies the resource ID of the central Purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Purview/accounts/{purview-name}` |
+| purviewManagedStorageId | Specifies the resource ID of the managed storage of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.Storage/storageAccounts/{storage-account-name}` |
+| purviewManagedEventHubId | Specifies the resource ID of the managed event hub of the central purview instance. | `/subscriptions/{subscription-id}/resourceGroups/{rg-name}/providers/Microsoft.EventHub/namespaces/{eventhub-namespace-name}` |
 | purviewSelfHostedIntegrationRuntimeAuthKey | Specifies the Auth Key for the Self-hosted integration runtime of Purview. | `<your-purview-shir-auth-key>` |
 | deploySelfHostedIntegrationRuntimes | Specifies whether the self-hosted integration runtimes should be deployed. This only works, if the pwsh script was uploded and is available. | `true` or `false` |
 | portalDeployment | Specifies whether the deployment was submitted through the Azure Portal. | `true` or `false` |

diff --git a/docs/reference/portal.dataLandingZone.json b/docs/reference/portal.dataLandingZone.json
@@ -426,13 +426,13 @@
                             ]
                         },
                         {
-                            "name": "dataGovernance",
-                            "label": "Data Governance",
+                            "name": "dataGovernanceSettings",
+                            "label": "Data Governance Settings",
                             "type": "Microsoft.Common.Section",
                             "visible": true,
                             "elements": [
                                 {
-                                    "name": "dataGovernanceText",
+                                    "name": "dataGovernanceSettingsText",
                                     "type": "Microsoft.Common.TextBlock",
                                     "visible": true,
                                     "options": {
@@ -475,6 +475,14 @@
                                     "options": {
                                         "hideConfirmation": true
                                     }
+                                },
+                                {
+                                    "name": "purviewApi",
+                                    "type": "Microsoft.Solutions.ArmApiControl",
+                                    "request": {
+                                        "method": "GET",
+                                        "path": "[concat(steps('generalSettings').dataGovernanceSettings.purviewId.id, '?api-version=2021-07-01')]"
+                                    }
                                 }
                             ]
                         }
@@ -1366,8 +1374,10 @@
                 "administratorPassword": "[if(empty(steps('generalSettings').servicesSettings.administratorPassword.password), '', steps('generalSettings').servicesSettings.administratorPassword.password)]",
                 "portalDeployment": true,
                 "deploySelfHostedIntegrationRuntimes": "[steps('generalSettings').servicesSettings.deploySelfHostedIntegrationRuntimes]",
-                "purviewSelfHostedIntegrationRuntimeAuthKey": "[if(empty(steps('generalSettings').dataGovernance.purviewSelfHostedIntegrationRuntimeAuthKey), '', steps('generalSettings').dataGovernance.purviewSelfHostedIntegrationRuntimeAuthKey)]",
-                "purviewId": "[if(empty(steps('generalSettings').dataGovernance.purviewId.id), '', steps('generalSettings').dataGovernance.purviewId.id)]",
+                "purviewSelfHostedIntegrationRuntimeAuthKey": "[if(empty(steps('generalSettings').dataGovernanceSettings.purviewSelfHostedIntegrationRuntimeAuthKey), '', steps('generalSettings').dataGovernanceSettings.purviewSelfHostedIntegrationRuntimeAuthKey)]",
+                "purviewId": "[if(empty(steps('generalSettings').dataGovernanceSettings.purviewId.id), '', steps('generalSettings').dataGovernanceSettings.purviewId.id)]",
+                "purviewManagedStorageId": "[if(empty(steps('generalSettings').dataGovernanceSettings.purviewId.id), '', steps('generalSettings').dataGovernanceSettings.purviewApi.properties.managedResources.storageAccount)]",
+                "purviewManagedEventHubId": "[if(empty(steps('generalSettings').dataGovernanceSettings.purviewId.id), '', steps('generalSettings').dataGovernanceSettings.purviewApi.properties.managedResources.eventHubNamespace)]",
                 "privateDnsZoneIdKeyVault": "[if(empty(steps('connectivitySettings').privateDnsZones.privateDnsZoneIdKeyVault), '', steps('connectivitySettings').privateDnsZones.privateDnsZoneIdKeyVault)]",
                 "privateDnsZoneIdDataFactory": "[if(empty(steps('connectivitySettings').privateDnsZones.privateDnsZoneIdDataFactory), '', steps('connectivitySettings').privateDnsZones.privateDnsZoneIdDataFactory)]",
                 "privateDnsZoneIdDataFactoryPortal": "[if(empty(steps('connectivitySettings').privateDnsZones.privateDnsZoneIdDataFactoryPortal), '', steps('connectivitySettings').privateDnsZones.privateDnsZoneIdDataFactoryPortal)]",

diff --git a/infra/main.bicep b/infra/main.bicep
@@ -58,6 +58,10 @@ param dnsServerAdresses array = [
 param administratorPassword string
 @description('Specifies the resource ID of the central purview instance.')
 param purviewId string = ''
+@description('Specifies the resource ID of the managed storage of the central purview instance.')
+param purviewManagedStorageId string = ''
+@description('Specifies the resource ID of the managed event hub of the central purview instance.')
+param purviewManagedEventHubId string = ''
 @secure()
 @description('Specifies the Auth Key for the Self-hosted integration runtime of Purview.')
 param purviewSelfHostedIntegrationRuntimeAuthKey string = ''
@@ -286,6 +290,8 @@ module sharedIntegrationServices 'modules/sharedintegration.bicep' = {
     sqlServer001Id: metadataServices.outputs.sqlServer001Id
     sqlDatabase001Name: metadataServices.outputs.sqlServer001DatabaseName
     purviewId: purviewId
+    purviewManagedStorageId: purviewManagedStorageId
+    purviewManagedEventHubId: purviewManagedEventHubId
     privateDnsZoneIdDataFactory: privateDnsZoneIdDataFactory
     privateDnsZoneIdDataFactoryPortal: privateDnsZoneIdDataFactoryPortal
     privateDnsZoneIdEventhubNamespace: privateDnsZoneIdEventhubNamespace

diff --git a/infra/main.json b/infra/main.json
@@ -5,7 +5,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.4.1008.15138",
-      "templateHash": "16528393280050774061"
+      "templateHash": "17725145245715892859"
     }
   },
   "parameters": {
@@ -155,6 +155,20 @@
         "description": "Specifies the resource ID of the central purview instance."
       }
     },
+    "purviewManagedStorageId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Specifies the resource ID of the managed storage of the central purview instance."
+      }
+    },
+    "purviewManagedEventHubId": {
+      "type": "string",
+      "defaultValue": "",
+      "metadata": {
+        "description": "Specifies the resource ID of the managed event hub of the central purview instance."
+      }
+    },
     "purviewSelfHostedIntegrationRuntimeAuthKey": {
       "type": "secureString",
       "defaultValue": "",
@@ -5169,6 +5183,12 @@
           "purviewId": {
             "value": "[parameters('purviewId')]"
           },
+          "purviewManagedStorageId": {
+            "value": "[parameters('purviewManagedStorageId')]"
+          },
+          "purviewManagedEventHubId": {
+            "value": "[parameters('purviewManagedEventHubId')]"
+          },
           "privateDnsZoneIdDataFactory": {
             "value": "[parameters('privateDnsZoneIdDataFactory')]"
           },
@@ -5186,7 +5206,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.4.1008.15138",
-              "templateHash": "2196576686803788151"
+              "templateHash": "17430668175094074912"
             }
           },
           "parameters": {
@@ -5221,6 +5241,14 @@
               "type": "string",
               "defaultValue": ""
             },
+            "purviewManagedStorageId": {
+              "type": "string",
+              "defaultValue": ""
+            },
+            "purviewManagedEventHubId": {
+              "type": "string",
+              "defaultValue": ""
+            },
             "storageRawId": {
               "type": "string"
             },
@@ -5567,6 +5595,12 @@
                   "purviewId": {
                     "value": "[parameters('purviewId')]"
                   },
+                  "purviewManagedStorageId": {
+                    "value": "[parameters('purviewManagedStorageId')]"
+                  },
+                  "purviewManagedEventHubId": {
+                    "value": "[parameters('purviewManagedEventHubId')]"
+                  },
                   "storageRawId": {
                     "value": "[parameters('storageRawId')]"
                   },
@@ -5596,7 +5630,7 @@
                     "_generator": {
                       "name": "bicep",
                       "version": "0.4.1008.15138",
-                      "templateHash": "5858292395769445241"
+                      "templateHash": "8237231481506109378"
                     }
                   },
                   "parameters": {
@@ -5624,6 +5658,14 @@
                       "type": "string",
                       "defaultValue": ""
                     },
+                    "purviewManagedStorageId": {
+                      "type": "string",
+                      "defaultValue": ""
+                    },
+                    "purviewManagedEventHubId": {
+                      "type": "string",
+                      "defaultValue": ""
+                    },
                     "storageRawId": {
                       "type": "string"
                     },
@@ -5799,6 +5841,66 @@
                         "[resourceId('Microsoft.DataFactory/factories/managedVirtualNetworks', parameters('datafactoryName'), 'default')]"
                       ]
                     },
+                    {
+                      "condition": "[not(empty(parameters('purviewId')))]",
+                      "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
+                      "apiVersion": "2018-06-01",
+                      "name": "[format('{0}/{1}/{2}', parameters('datafactoryName'), 'default', 'Purview')]",
+                      "properties": {
+                        "fqdns": [],
+                        "groupId": "account",
+                        "privateLinkResourceId": "[parameters('purviewId')]"
+                      },
+                      "dependsOn": [
+                        "[resourceId('Microsoft.DataFactory/factories', parameters('datafactoryName'))]",
+                        "[resourceId('Microsoft.DataFactory/factories/managedVirtualNetworks', parameters('datafactoryName'), 'default')]"
+                      ]
+                    },
+                    {
+                      "condition": "[not(empty(parameters('purviewManagedStorageId')))]",
+                      "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
+                      "apiVersion": "2018-06-01",
+                      "name": "[format('{0}/{1}/{2}', parameters('datafactoryName'), 'default', 'Purview_blob')]",
+                      "properties": {
+                        "fqdns": [],
+                        "groupId": "blob",
+                        "privateLinkResourceId": "[parameters('purviewManagedStorageId')]"
+                      },
+                      "dependsOn": [
+                        "[resourceId('Microsoft.DataFactory/factories', parameters('datafactoryName'))]",
+                        "[resourceId('Microsoft.DataFactory/factories/managedVirtualNetworks', parameters('datafactoryName'), 'default')]"
+                      ]
+                    },
+                    {
+                      "condition": "[not(empty(parameters('purviewManagedStorageId')))]",
+                      "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
+                      "apiVersion": "2018-06-01",
+                      "name": "[format('{0}/{1}/{2}', parameters('datafactoryName'), 'default', 'Purview_queue')]",
+                      "properties": {
+                        "fqdns": [],
+                        "groupId": "queue",
+                        "privateLinkResourceId": "[parameters('purviewManagedStorageId')]"
+                      },
+                      "dependsOn": [
+                        "[resourceId('Microsoft.DataFactory/factories', parameters('datafactoryName'))]",
+                        "[resourceId('Microsoft.DataFactory/factories/managedVirtualNetworks', parameters('datafactoryName'), 'default')]"
+                      ]
+                    },
+                    {
+                      "condition": "[not(empty(parameters('purviewManagedEventHubId')))]",
+                      "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
+                      "apiVersion": "2018-06-01",
+                      "name": "[format('{0}/{1}/{2}', parameters('datafactoryName'), 'default', 'Purview_namespace')]",
+                      "properties": {
+                        "fqdns": [],
+                        "groupId": "namespace",
+                        "privateLinkResourceId": "[parameters('purviewManagedEventHubId')]"
+                      },
+                      "dependsOn": [
+                        "[resourceId('Microsoft.DataFactory/factories', parameters('datafactoryName'))]",
+                        "[resourceId('Microsoft.DataFactory/factories/managedVirtualNetworks', parameters('datafactoryName'), 'default')]"
+                      ]
+                    },
                     {
                       "type": "Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints",
                       "apiVersion": "2018-06-01",

diff --git a/infra/modules/services/datafactorysharedintegration.bicep b/infra/modules/services/datafactorysharedintegration.bicep
@@ -13,6 +13,8 @@ param privateDnsZoneIdDataFactory string = ''
 param privateDnsZoneIdDataFactoryPortal string = ''
 
 param purviewId string = ''
+param purviewManagedStorageId string = ''
+param purviewManagedEventHubId string = ''
 param storageRawId string
 param storageEnrichedCuratedId string
 param databricks001Id string
@@ -149,6 +151,46 @@ resource datafactoryManagedIntegrationRuntime001 'Microsoft.DataFactory/factorie
   }
 }
 
+resource datafactoryPurviewManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if(!empty(purviewId)) {
+  parent: datafactoryManagedVirtualNetwork
+  name: 'Purview'
+  properties: {
+    fqdns: []
+    groupId: 'account'
+    privateLinkResourceId: purviewId
+  }
+}
+
+resource datafactoryPurviewBlobManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if(!empty(purviewManagedStorageId)) {
+  parent: datafactoryManagedVirtualNetwork
+  name: 'Purview_blob'
+  properties: {
+    fqdns: []
+    groupId: 'blob'
+    privateLinkResourceId: purviewManagedStorageId
+  }
+}
+
+resource datafactoryPurviewQueueManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if(!empty(purviewManagedStorageId)) {
+  parent: datafactoryManagedVirtualNetwork
+  name: 'Purview_queue'
+  properties: {
+    fqdns: []
+    groupId: 'queue'
+    privateLinkResourceId: purviewManagedStorageId
+  }
+}
+
+resource datafactoryPurviewNamespaceManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = if(!empty(purviewManagedEventHubId)) {
+  parent: datafactoryManagedVirtualNetwork
+  name: 'Purview_namespace'
+  properties: {
+    fqdns: []
+    groupId: 'namespace'
+    privateLinkResourceId: purviewManagedEventHubId
+  }
+}
+
 resource datafactoryKeyVault001ManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = {
   parent: datafactoryManagedVirtualNetwork
   name: replace(keyVault001Name, '-', '')

diff --git a/infra/modules/sharedintegration.bicep b/infra/modules/sharedintegration.bicep
@@ -16,6 +16,8 @@ param databricksIntegration001PrivateSubnetName string
 param databricksIntegration001PublicSubnetName string
 param subnetId string
 param purviewId string = ''
+param purviewManagedStorageId string = ''
+param purviewManagedEventHubId string = ''
 param storageRawId string
 param storageEnrichedCuratedId string
 param keyVault001Id string
@@ -73,6 +75,8 @@ module datafactoryIntegration001 'services/datafactorysharedintegration.bicep' =
     privateDnsZoneIdDataFactory: privateDnsZoneIdDataFactory
     privateDnsZoneIdDataFactoryPortal: privateDnsZoneIdDataFactoryPortal
     purviewId: purviewId
+    purviewManagedStorageId: purviewManagedStorageId
+    purviewManagedEventHubId: purviewManagedEventHubId
     storageRawId: storageRawId
     storageEnrichedCuratedId: storageEnrichedCuratedId
     databricks001Id: databricksIntegration001.outputs.databricksId

diff --git a/infra/params.dev.json b/infra/params.dev.json
@@ -64,6 +64,12 @@
         "purviewId": {
             "value": "/subscriptions/17588eb2-2943-461a-ab3f-00a3ceac3112/resourceGroups/dmz-dev-governance/providers/Microsoft.Purview/accounts/dmz-dev-purview001"
         },
+        "purviewManagedStorageId": {
+            "value": "/subscriptions/17588eb2-2943-461a-ab3f-00a3ceac3112/resourceGroups/dmz-dev-purview001/providers/Microsoft.Storage/storageAccounts/scannortheuropekkkshlo"
+        },
+        "purviewManagedEventHubId": {
+            "value": "/subscriptions/17588eb2-2943-461a-ab3f-00a3ceac3112/resourceGroups/dmz-dev-purview001/providers/Microsoft.EventHub/namespaces/Atlas-a487e969-df29-4159-8858-9e68d81285e9"
+        },
         "purviewSelfHostedIntegrationRuntimeAuthKey": {
             "value": ""
         },