Network & AAD Integrated API Management (#112)

* Added APIM * AppGW -> APIM integration * Use SSL root cert * Test APIM in pipeline * Feature toggle * Updated vm extension versions * Allow CI build to inherit state * re-order VM extensions * APIM AAD AuthN support
geekzter · Sep 17, 2020 · 57efb65 · 57efb65
1 parent 8ea9d5b
commit 57efb65
Show file tree

Hide file tree

Showing 21 changed files with 1,335 additions and 198 deletions.
diff --git a/kusto/nsg-statistics.kql b/kusto/nsg-statistics.kql
@@ -0,0 +1,16 @@
+AzureNetworkAnalytics_CL                                    
+| where SubType_s == 'FlowLog' and  (FASchemaVersion_s == '1' or FASchemaVersion_s == '2')  and  FlowStartTime_t between (datetime('2020-09-02T09:41:39.000Z') .. datetime('2020-09-03T09:41:39.000Z'))                                      
+| extend completeNsgRule = strcat(NSGList_s, '/', NSGRule_s)                                                                        
+| extend isMaliciousFlow = (FlowType_s == 'MaliciousFlow'),                                              isBlocked = (FlowStatus_s == 'D')                                     
+| extend AllowedInbound = tolong(iif(FlowDirection_s == 'I' and FlowStatus_s == 'A', tolong(AllowedInFlows_d), 0)),                     BlockedInbound = tolong(iif(FlowDirection_s == 'I' and FlowStatus_s == 'D', tolong(DeniedInFlows_d), 0)),                      AllowedOutbound = tolong(iif(FlowDirection_s == 'O' and FlowStatus_s == 'A', tolong(AllowedOutFlows_d), 0)),                      BlockedOutbound = tolong(iif(FlowDirection_s == 'O' and FlowStatus_s == 'D', tolong(DeniedOutFlows_d), 0))                                     
+| extend AllowedInboundBytes = iif(not(isBlocked), tolong(InboundBytes_d), 0),                      BlockedInboundBytes = iif(isBlocked, tolong(InboundBytes_d), 0),                      AllowedOutboundBytes = iif(not(isBlocked), tolong(OutboundBytes_d), 0),                      BlockedOutboundBytes = iif(isBlocked, tolong(OutboundBytes_d), 0)                                     
+| extend AllowedInboundPackets = iif(not(isBlocked), tolong(InboundPackets_d), 0),                      BlockedInboundPackets = iif(isBlocked, tolong(InboundPackets_d), 0),                      AllowedOutboundPackets = iif(not(isBlocked), tolong(OutboundPackets_d), 0),                      BlockedOutboundPackets = iif(isBlocked, tolong(OutboundPackets_d), 0)                                     
+| extend MaliciousAllowedInboundFlows = iif(isMaliciousFlow, AllowedInbound, 0),                          MaliciousBlockedInboundFlows = iif(isMaliciousFlow, BlockedInbound, 0),                          MaliciousAllowedOutboundFlows = iif(isMaliciousFlow, AllowedOutbound, 0),                          MaliciousBlockedOutboundFlows = iif(isMaliciousFlow, BlockedOutbound, 0)                                    
+| extend MaliciousAllowedInboundBytes = iif(isMaliciousFlow, AllowedInboundBytes, 0),                          MaliciousBlockedInboundBytes = iif(isMaliciousFlow, BlockedInboundBytes, 0),                          MaliciousAllowedOutboundBytes = iif(isMaliciousFlow, AllowedOutboundBytes, 0),                          MaliciousBlockedOutboundBytes = iif(isMaliciousFlow, BlockedOutboundBytes, 0)                                    
+| extend MaliciousAllowedInboundPackets = iif(isMaliciousFlow, AllowedInboundPackets, 0),                          MaliciousBlockedInboundPackets = iif(isMaliciousFlow, BlockedInboundPackets, 0),                          MaliciousAllowedOutboundPackets = iif(isMaliciousFlow, AllowedOutboundPackets, 0),                          MaliciousBlockedOutboundPackets = iif(isMaliciousFlow, BlockedOutboundPackets, 0)                                    
+| summarize totalAllowedInboundFlows = sum(AllowedInbound),                 totalBlockedInboundFlows = sum(BlockedInbound),                 totalAllowedOutboundFlows = sum(AllowedOutbound),                 totalBlockedOutboundFlows = sum(BlockedOutbound),                totalMaliciousAllowedInboundFlows = sum(MaliciousAllowedInboundFlows),                 totalMaliciousBlockedInboundFlows = sum(MaliciousBlockedInboundFlows),                 totalMaliciousAllowedOutboundFlows = sum(MaliciousAllowedOutboundFlows),                 totalMaliciousBlockedOutboundFlows = sum(MaliciousBlockedOutboundFlows),                                                totalAllowedInboundBytes = sum(AllowedInboundBytes),                 totalBlockedInboundBytes = sum(BlockedInboundBytes),                 totalAllowedOutboundBytes = sum(AllowedOutboundBytes),                 totalBlockedOutboundBytes = sum(BlockedOutboundBytes),                totalMaliciousAllowedInboundBytes = sum(MaliciousAllowedInboundBytes),                 totalMaliciousBlockedInboundBytes= sum(MaliciousBlockedInboundBytes),                 totalMaliciousAllowedOutboundBytes = sum(MaliciousAllowedOutboundBytes),                 totalMaliciousBlockedOutboundBytes = sum(MaliciousBlockedOutboundBytes),                                                totalAllowedInboundPackets = sum(AllowedInboundPackets),                 totalBlockedInboundPackets = sum(BlockedInboundPackets),                 totalAllowedOutboundPackets = sum(AllowedOutboundPackets),                 totalBlockedOutboundPackets = sum(BlockedOutboundPackets),                totalMaliciousAllowedInboundPackets = sum(MaliciousAllowedInboundPackets),                 totalMaliciousBlockedInboundPackets = sum(MaliciousBlockedInboundPackets),                 totalMaliciousAllowedOutboundPackets = sum(MaliciousAllowedOutboundPackets),                 totalMaliciousBlockedOutboundPackets = sum(MaliciousBlockedOutboundPackets) by NSGList_s                                     
+| extend TotalFlows = totalAllowedInboundFlows + totalBlockedInboundFlows + totalAllowedOutboundFlows + totalBlockedOutboundFlows                                    
+| extend TotalBytes = totalAllowedInboundBytes + totalBlockedInboundBytes + totalAllowedOutboundBytes + totalBlockedOutboundBytes                                    
+| extend TotalPackets = totalAllowedInboundPackets + totalBlockedInboundPackets + totalAllowedOutboundPackets + totalBlockedOutboundPackets                                    
+| where TotalFlows > 0                                    
+| sort by TotalFlows
diff --git a/pipelines/templates/vdc-terraform-apply.yml b/pipelines/templates/vdc-terraform-apply.yml
@@ -212,6 +212,12 @@ stages:
           inputs:
             secureFile: '$(TF_VAR_vanity_certificate_path)'
 
+        - task: DownloadSecureFile@1
+          displayName: 'Download SSL root certificate'
+          enabled: ${{ not(parameters.testMode) }}
+          inputs:
+            secureFile: '$(TF_VAR_vanity_root_certificate_cer_path)'
+
         - task: DownloadSecureFile@1
           displayName: 'Download VPN root certificate'
           enabled: ${{ not(parameters.testMode) }}
@@ -304,6 +310,12 @@ stages:
           inputs:
             secureFile: '$(TF_VAR_vanity_certificate_path)'
 
+        - task: DownloadSecureFile@1
+          displayName: 'Download SSL root certificate'
+          enabled: ${{ not(parameters.testMode) }}
+          inputs:
+            secureFile: '$(TF_VAR_vanity_root_certificate_cer_path)'
+
         - task: DownloadSecureFile@1
           displayName: 'Download VPN root certificate'
           enabled: ${{ not(parameters.testMode) }}
@@ -719,13 +731,23 @@ stages:
         - Deploy
         - DeployEnvironment
         - Provision
+        - ${{ if eq(parameters.reprovision, true) }}:
+          - Reprovision
       displayName: 'Test Applications'
       condition: succeeded()
       timeoutInMinutes: $[ variables['jobTimeOutMinutes'] ]
 
       variables:
-        iaasAppUrl: $[ dependencies.Provision.outputs['terraform.iaas_app_url'] ]
-        paasAppUrl: $[ dependencies.Provision.outputs['terraform.paas_app_url'] ]
+        apimGWUrl:     $[ dependencies.Provision.outputs['terraform.apim_gateway_url'] ]
+        apimPortalUrl: $[ dependencies.Provision.outputs['terraform.apim_developer_portal_url'] ]
+        deployAPIM:    $[ eq(variables['TF_VAR_deploy_api_gateway'], 'true') ]
+        iaasAppUrl:    $[ dependencies.Provision.outputs['terraform.iaas_app_url'] ]
+        paasAppUrl:    $[ dependencies.Provision.outputs['terraform.paas_app_url'] ]
+        # apim_demo_api_key get's updated on every terraform apply, make sure we get up to date data
+        ${{ if eq(parameters.reprovision, false) }}:
+          apimDemoKey: $[ dependencies.Provision.outputs['terraform.apim_demo_api_key'] ]
+        ${{ if eq(parameters.reprovision, true) }}:
+          apimDemoKey: $[ dependencies.Reprovision.outputs['terraform.apim_demo_api_key'] ]
 
       pool:
         vmImage: 'ubuntu-latest'
@@ -751,6 +773,7 @@ stages:
               Start-Sleep -Seconds 10
               
               # Request page
+              Write-Host "Requesting $(iaasAppUrl)..."
               Invoke-WebRequest -Uri $(iaasAppUrl) -MaximumRetryCount 9
             failOnStderr: true
             pwsh: true
@@ -766,10 +789,38 @@ stages:
               Start-Sleep -Seconds 10
               
               # Request page
+              Write-Host "Requesting $(paasAppUrl)..."
               Invoke-WebRequest -Uri $(paasAppUrl) -MaximumRetryCount 9
             failOnStderr: true
             pwsh: true
 
+        # Test API Management
+        - task: PowerShell@2
+          displayName: 'Test API Management Portal'
+          condition: and(succeeded(), eq(variables.deployAPIM, true))
+          enabled: ${{ not(parameters.testMode) }}
+          inputs:
+            targetType: 'inline'
+            script: |
+              # Request page
+              Write-Host "Requesting $(apimPortalUrl)..."
+              Invoke-WebRequest -Uri $(apimPortalUrl) -MaximumRetryCount 9
+            failOnStderr: true
+            pwsh: true
+        - task: PowerShell@2
+          displayName: 'Test API Management Gateway'
+          condition: and(succeeded(), eq(variables.deployAPIM, true))
+          enabled: ${{ not(parameters.testMode) }}
+          inputs:
+            targetType: 'inline'
+            script: |
+              # Call REST Api
+              $apimDemoUrl = "$(apimGWUrl)echo/resource?param1=sample"
+              Write-Host "Performing GET request of $apimDemoUrl with Header Ocp-Apim-Subscription-Key = $(apimDemoKey)..."
+              Invoke-RestMethod -Header @{ "Ocp-Apim-Subscription-Key" = "$(apimDemoKey)" } -Method Get -Uri $apimDemoUrl -MaximumRetryCount 9
+            failOnStderr: true
+            pwsh: true
+
   - ${{ if ne(parameters.destroy, 'Never') }}:
     # Destroy infrastructure
     - job: 'Destroy'
@@ -816,6 +867,12 @@ stages:
           inputs:
             secureFile: '$(TF_VAR_vanity_certificate_path)'
 
+        - task: DownloadSecureFile@1
+          displayName: 'Download SSL root certificate'
+          enabled: ${{ not(parameters.testMode) }}
+          inputs:
+            secureFile: '$(TF_VAR_vanity_root_certificate_cer_path)'
+
         - task: DownloadSecureFile@1
           displayName: 'Download VPN root certificate'
           enabled: ${{ not(parameters.testMode) }}

diff --git a/pipelines/vdc-terraform-apply-ci.yml b/pipelines/vdc-terraform-apply-ci.yml
@@ -13,12 +13,17 @@ parameters:
   - ci1
   - ci2
   - ci3
+- name: inherit
+  displayName: Inherit state
+  type: boolean
+  default: false
 - name: destroy
   displayName: Destroy infrastructure
   type: string
   default: Always
   values:
   - Always
+  - Never
   - 'On failure'
   - 'On success'
 - name: unpinTerraformProviders
@@ -68,7 +73,7 @@ stages:
   parameters:
     name: 'CI'
     displayName: 'Provision, Deploy, Tear down'
-    inherit: false
+    inherit: ${{ parameters.inherit }}
     provision: true
     reprovision: true
     deploy: true

diff --git a/pipelines/vdc-terraform-plan-ci.yml b/pipelines/vdc-terraform-plan-ci.yml
@@ -73,6 +73,11 @@ jobs:
       inputs:
         secureFile: '$(TF_VAR_vanity_certificate_path)'
 
+    - task: DownloadSecureFile@1
+      displayName: 'Download SSL root certificate'
+      inputs:
+        secureFile: '$(TF_VAR_vanity_root_certificate_cer_path)'
+
     - task: DownloadSecureFile@1
       displayName: 'Download VPN root certificate'
       inputs:

diff --git a/scripts/host/configure_mgmtvm_roles.ps1 b/scripts/host/configure_mgmtvm_roles.ps1
@@ -5,6 +5,7 @@
 .DESCRIPTION 
     This script is downloaded and executed using the custom script VM extension
 #> 
+#Set-PSDebug -Trace 1 # Trace in the case the extension doesn't successfully load
 
 $privateDNS = "168.63.129.16" # Azure Private DNS
 $publicDNS = @("8.8.8.8","8.8.4.4") # Google Public DNS

diff --git a/scripts/host/private_link_zones.conf b/scripts/host/private_link_zones.conf
@@ -3,60 +3,57 @@
 # https://docs.microsoft.com/en-us/azure/private-link/private-endpoint-dns
 azure-automation.net
 database.windows.net
-database.windows.net
 blob.core.windows.net
 table.core.windows.net
 queue.core.windows.net
-file.core.windows.net
-web.core.windows.net
-dfs.core.windows.net
-documents.azure.com
-mongo.cosmos.azure.com
-cassandra.cosmos.azure.com
-gremlin.cosmos.azure.com
-table.cosmos.azure.com
-postgres.database.azure.com
-mysql.database.azure.com
-mariadb.database.azure.com
+# file.core.windows.net
+# web.core.windows.net
+# dfs.core.windows.net
+# documents.azure.com
+# mongo.cosmos.azure.com
+# cassandra.cosmos.azure.com
+# gremlin.cosmos.azure.com
+# table.cosmos.azure.com
+# postgres.database.azure.com
+# mysql.database.azure.com
+# mariadb.database.azure.com
 vault.azure.net
 vaultcore.azure.net
-westeurope.azmk8s.io
-northeurope.azmk8s.io
-uksouth.azmk8s.io
-ukwest.azmk8s.io
-southeastasia.azmk8s.io
-eastasia.azmk8s.io
-centralus.azmk8s.io
-eastus.azmk8s.io
-eastus2.azmk8s.io
-westsus.azmk8s.io
-westus2.azmk8s.io
-search.windows.net
+# westeurope.azmk8s.io
+# northeurope.azmk8s.io
+# uksouth.azmk8s.io
+# ukwest.azmk8s.io
+# southeastasia.azmk8s.io
+# eastasia.azmk8s.io
+# centralus.azmk8s.io
+# eastus.azmk8s.io
+# eastus2.azmk8s.io
+# westsus.azmk8s.io
+# westus2.azmk8s.io
+# search.windows.net
 azurecr.io
-azconfig.io
-westeurope.backup.windowsazure.com
-northeurope.backup.windowsazure.com
-uksouth.backup.windowsazure.com
-ukwest.backup.windowsazure.com
-southeastasia.backup.windowsazure.com
-eastasia.backup.windowsazure.com
-centralus.backup.windowsazure.com
-eastus.backup.windowsazure.com
-eastus2.backup.windowsazure.com
-westsus.backup.windowsazure.com
-westus2.backup.windowsazure.com
-servicebus.windows.net
-servicebus.windows.net
-azure-devices.net
+# azconfig.io
+# westeurope.backup.windowsazure.com
+# northeurope.backup.windowsazure.com
+# uksouth.backup.windowsazure.com
+# ukwest.backup.windowsazure.com
+# southeastasia.backup.windowsazure.com
+# eastasia.backup.windowsazure.com
+# centralus.backup.windowsazure.com
+# eastus.backup.windowsazure.com
+# eastus2.backup.windowsazure.com
+# westsus.backup.windowsazure.com
+# westus2.backup.windowsazure.com
 servicebus.windows.net
-eventgrid.azure.net
+# azure-devices.net
+# eventgrid.azure.net
 azurewebsites.net
-api.azureml.ms
-azure-devices.net
-service.signalr.net
+# api.azureml.ms
+# azure-devices.net
+# service.signalr.net
 monitor.azure.com
 oms.opinsights.azure.com
 ods.opinsights.azure.com
 agentsvc.azure-automation.com
-cognitiveservices.azure.com
-afs.azure.net
+# cognitiveservices.azure.com
+# afs.azure.net
diff --git a/scripts/templatize_dashboard.ps1 b/scripts/templatize_dashboard.ps1
@@ -100,6 +100,8 @@ if ($prefix -and $deploymentName -and $suffix) {
 if ($appRGShort) {
     $template = $template -Replace "${appRGShort}", "`$`{paas_app_resource_group_short`}"
 }
+$template = $template -Replace "http[s?]://[\w\.]*apiproxy[^`"]*/", "`$`{apim_gw_url`}"
+$template = $template -Replace "http[s?]://[\w\.]*apiportal[\w\.]*/", "`$`{apim_portal_url`}"
 $template = $template -Replace "http[s?]://[\w\.]*iisapp[\w\.]*/", "`$`{iaas_app_url`}"
 $template = $template -Replace "http[s?]://[\w\.]*webapp[\w\.]*/", "`$`{paas_app_url`}"
 $template = $template -Replace "https://dev.azure.com[^`']*_build[^`']*`'", "`$`{build_web_url`}`'"