From 82514bdb0d361b461af6f24e62e1035aed63dbca Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Wed, 29 Sep 2021 13:48:51 -0700 Subject: [PATCH 1/5] Changing inline bash for stress test resource deployment --- .../TestResources/New-TestResources.ps1 | 42 ++++++++++++++++--- .../deploy-stress-test-resources.ps1 | 35 ++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 eng/common/TestResources/deploy-stress-test-resources.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 189f5185c71e..edce856b33d8 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -76,7 +76,14 @@ param ( [switch] $Force, [Parameter()] - [switch] $OutFile + [switch] $OutFile, + + [Parameter()] + [boolean] $OutFileEncryption = $true, + + [Parameter()] + [ValidateSet('json', 'dotenv', 'pwshenv')] + [string] $OutFileFormat = 'json' ) # By default stop for any error. @@ -159,7 +166,14 @@ try { # Enumerate test resources to deploy. Fail if none found. $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path - $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path + + $root = '' + if (Split-Path -IsAbsolute $ServiceDirectory) { + $root = $ServiceDirectory + } else { + $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path + } + $templateFiles = @() 'test-resources.json', 'test-resources.bicep' | ForEach-Object { @@ -598,12 +612,28 @@ try { } $outputFile = "$($templateFile.originalFilePath).env" + if ($env:ENV_FILE) { + $outputFile = "$env:ENV_FILE" + } - $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) - $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + $environmentText = '' + if ($OutFileFormat -eq 'json') { + $environmentText = $deploymentOutputs | ConvertTo-Json; + } elseif ($OutFileFormat -eq 'dotenv') { + foreach($entry in $deploymentOutputs.GetEnumerator()) { + $environmentText += "$($entry.name)=$($entry.value)`n" + } + } elseif ($OutFileFormat -eq 'pwshenv') { + foreach($entry in $deploymentOutputs.GetEnumerator()) { + $environmentText += "`$$($entry.name)=$($entry.value)`n" + } + } - Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force + $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) + if ($OutFileEncryption) { + $bytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + } + Set-Content $outputFile -Value $bytes -AsByteStream -Force Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" } else { diff --git a/eng/common/TestResources/deploy-stress-test-resources.ps1 b/eng/common/TestResources/deploy-stress-test-resources.ps1 new file mode 100644 index 000000000000..64be64321f26 --- /dev/null +++ b/eng/common/TestResources/deploy-stress-test-resources.ps1 @@ -0,0 +1,35 @@ +$secrets = @{} +$secretsDir = "/mnt/secrets/static/*" +Get-ChildItem -Path $secretsDir | ForEach-Object { + foreach($line in Get-Content $_) { + $idx = $line.IndexOf("=") + if ($idx -gt 0) { + $key = $line.Substring(0, $idx) + $val = $line.Substring($idx + 1) + $secrets.Add($key, $val) + } + } +} + +$templateParams = Get-Content "/mnt/azure/parameters.json" | ConvertFrom-json -AsHashTable + +& $PSScriptRoot/New-TestResources.ps1 ` + -BaseName $env:BASE_NAME ` + -SubscriptionId $secrets.AZURE_SUBSCRIPTION_ID ` + -TenantId $secrets.AZURE_TENANT_ID ` + -ProvisionerApplicationId $secrets.AZURE_CLIENT_ID ` + -ProvisionerApplicationSecret $secrets.AZURE_CLIENT_SECRET ` + -TestApplicationId $secrets.AZURE_CLIENT_ID ` + -TestApplicationSecret $secrets.AZURE_CLIENT_SECRET ` + -TestApplicationOid $secrets.AZURE_CLIENT_OID ` + -ArmTemplateParameters $templateParams ` + -Location 'westus2' ` + -DeleteAfterHours 168 ` + -ServiceDirectory '/mnt/azure/' ` + -CI ` + -Force ` + -OutFile ` + -OutFileEncryption $false ` + -OutFileFormat 'dotenv' + +#> \ No newline at end of file From 5412610eba04a8f3ba933028db0f1d281db21663 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 26 Oct 2021 14:32:47 -0700 Subject: [PATCH 2/5] PR-mod --- .../TestResources/New-TestResources.ps1 | 21 +++++-------------- .../deploy-stress-test-resources.ps1 | 12 +++++------ .../stress-testing/test-resources-post.ps1 | 13 ++++++++++++ 3 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 eng/common/scripts/stress-testing/test-resources-post.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index edce856b33d8..efe499d738c7 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -78,9 +78,6 @@ param ( [Parameter()] [switch] $OutFile, - [Parameter()] - [boolean] $OutFileEncryption = $true, - [Parameter()] [ValidateSet('json', 'dotenv', 'pwshenv')] [string] $OutFileFormat = 'json' @@ -166,14 +163,7 @@ try { # Enumerate test resources to deploy. Fail if none found. $repositoryRoot = "$PSScriptRoot/../../.." | Resolve-Path - - $root = '' - if (Split-Path -IsAbsolute $ServiceDirectory) { - $root = $ServiceDirectory - } else { - $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path - } - + $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | Resolve-Path $templateFiles = @() 'test-resources.json', 'test-resources.bicep' | ForEach-Object { @@ -629,11 +619,10 @@ try { } } - $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) - if ($OutFileEncryption) { - $bytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) - } - Set-Content $outputFile -Value $bytes -AsByteStream -Force + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) + $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) + + Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force Write-Host "Test environment settings`n $environmentText`nstored into encrypted $outputFile" } else { diff --git a/eng/common/TestResources/deploy-stress-test-resources.ps1 b/eng/common/TestResources/deploy-stress-test-resources.ps1 index 64be64321f26..81ec389ee9b1 100644 --- a/eng/common/TestResources/deploy-stress-test-resources.ps1 +++ b/eng/common/TestResources/deploy-stress-test-resources.ps1 @@ -11,7 +11,10 @@ Get-ChildItem -Path $secretsDir | ForEach-Object { } } -$templateParams = Get-Content "/mnt/azure/parameters.json" | ConvertFrom-json -AsHashTable +$templateParams = Get-Content "/mnt/testresources/parameters.json" | ConvertFrom-json -AsHashTable +mkdir /azure +Copy-Item "/common/scripts/stress-testing/test-resources-post.ps1" -Destination "/azure/" +Copy-Item "/mnt/testresources/*" -Destination "/azure/" & $PSScriptRoot/New-TestResources.ps1 ` -BaseName $env:BASE_NAME ` @@ -25,11 +28,8 @@ $templateParams = Get-Content "/mnt/azure/parameters.json" | ConvertFrom-json -A -ArmTemplateParameters $templateParams ` -Location 'westus2' ` -DeleteAfterHours 168 ` - -ServiceDirectory '/mnt/azure/' ` + -ServiceDirectory '/azure/' ` -CI ` - -Force ` - -OutFile ` - -OutFileEncryption $false ` - -OutFileFormat 'dotenv' + -Force #> \ No newline at end of file diff --git a/eng/common/scripts/stress-testing/test-resources-post.ps1 b/eng/common/scripts/stress-testing/test-resources-post.ps1 new file mode 100644 index 000000000000..0c03bfb30529 --- /dev/null +++ b/eng/common/scripts/stress-testing/test-resources-post.ps1 @@ -0,0 +1,13 @@ +param ( + [hashtable] $DeploymentOutputs +) + +$outputFile = "$env:ENV_FILE" + +$environmentText = '' +foreach($entry in $DeploymentOutputs.GetEnumerator()) { + $environmentText += "$($entry.name)=$($entry.value)`n" +} + +$bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) +Set-Content $outputFile -Value $bytes -AsByteStream -Force From dc4e1f91283a5f90d842cbe0b620eb7de95114d9 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Thu, 28 Oct 2021 13:14:51 -0700 Subject: [PATCH 3/5] pr-mod --- eng/common/TestResources/New-TestResources.ps1 | 2 +- eng/common/TestResources/deploy-stress-test-resources.ps1 | 7 ++++++- eng/common/scripts/stress-testing/test-resources-post.ps1 | 6 +++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index efe499d738c7..efcc3eb13cb0 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -338,7 +338,7 @@ try { $serviceName = if (Split-Path $ServiceDirectory) { Split-Path -Leaf $ServiceDirectory } else { - $ServiceDirectory + $ServiceDirectory.Trim('/') } $ResourceGroupName = if ($ResourceGroupName) { diff --git a/eng/common/TestResources/deploy-stress-test-resources.ps1 b/eng/common/TestResources/deploy-stress-test-resources.ps1 index 81ec389ee9b1..0d73f98f319f 100644 --- a/eng/common/TestResources/deploy-stress-test-resources.ps1 +++ b/eng/common/TestResources/deploy-stress-test-resources.ps1 @@ -1,3 +1,8 @@ +param ( + [Parameter()] + [string] $TemplateParameters +) + $secrets = @{} $secretsDir = "/mnt/secrets/static/*" Get-ChildItem -Path $secretsDir | ForEach-Object { @@ -11,7 +16,7 @@ Get-ChildItem -Path $secretsDir | ForEach-Object { } } -$templateParams = Get-Content "/mnt/testresources/parameters.json" | ConvertFrom-json -AsHashTable +$templateParams = Get-Content $TemplateParameters | ConvertFrom-json -AsHashTable mkdir /azure Copy-Item "/common/scripts/stress-testing/test-resources-post.ps1" -Destination "/azure/" Copy-Item "/mnt/testresources/*" -Destination "/azure/" diff --git a/eng/common/scripts/stress-testing/test-resources-post.ps1 b/eng/common/scripts/stress-testing/test-resources-post.ps1 index 0c03bfb30529..d71c03f48265 100644 --- a/eng/common/scripts/stress-testing/test-resources-post.ps1 +++ b/eng/common/scripts/stress-testing/test-resources-post.ps1 @@ -1,5 +1,9 @@ param ( - [hashtable] $DeploymentOutputs + [Parameter()] + [hashtable] $DeploymentOutputs, + + [Parameter(ValueFromRemainingArguments = $true)] + $RemainingArguments ) $outputFile = "$env:ENV_FILE" From fa5d9d5896bd64387b60858411e4915974bc765f Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Mon, 1 Nov 2021 12:38:15 -0700 Subject: [PATCH 4/5] pr-mod --- eng/common/TestResources/deploy-stress-test-resources.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/TestResources/deploy-stress-test-resources.ps1 b/eng/common/TestResources/deploy-stress-test-resources.ps1 index 0d73f98f319f..96a3ddcc8840 100644 --- a/eng/common/TestResources/deploy-stress-test-resources.ps1 +++ b/eng/common/TestResources/deploy-stress-test-resources.ps1 @@ -1,6 +1,6 @@ param ( [Parameter()] - [string] $TemplateParameters + [string] $TemplateParametersPath ) $secrets = @{} @@ -16,7 +16,7 @@ Get-ChildItem -Path $secretsDir | ForEach-Object { } } -$templateParams = Get-Content $TemplateParameters | ConvertFrom-json -AsHashTable +$templateParams = Get-Content $TemplateParametersPath | ConvertFrom-json -AsHashTable mkdir /azure Copy-Item "/common/scripts/stress-testing/test-resources-post.ps1" -Destination "/azure/" Copy-Item "/mnt/testresources/*" -Destination "/azure/" From 4c254ce4a94d9647fbcdd3d34ab23a547248d1c5 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 2 Nov 2021 14:51:29 -0700 Subject: [PATCH 5/5] pr-mod --- .../TestResources/New-TestResources.ps1 | 23 +---------- .../deploy-stress-test-resources.ps1 | 40 ------------------- .../stress-testing/test-resources-post.ps1 | 17 -------- 3 files changed, 2 insertions(+), 78 deletions(-) delete mode 100644 eng/common/TestResources/deploy-stress-test-resources.ps1 delete mode 100644 eng/common/scripts/stress-testing/test-resources-post.ps1 diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index efcc3eb13cb0..93801d6836a4 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -76,11 +76,7 @@ param ( [switch] $Force, [Parameter()] - [switch] $OutFile, - - [Parameter()] - [ValidateSet('json', 'dotenv', 'pwshenv')] - [string] $OutFileFormat = 'json' + [switch] $OutFile ) # By default stop for any error. @@ -602,23 +598,8 @@ try { } $outputFile = "$($templateFile.originalFilePath).env" - if ($env:ENV_FILE) { - $outputFile = "$env:ENV_FILE" - } - - $environmentText = '' - if ($OutFileFormat -eq 'json') { - $environmentText = $deploymentOutputs | ConvertTo-Json; - } elseif ($OutFileFormat -eq 'dotenv') { - foreach($entry in $deploymentOutputs.GetEnumerator()) { - $environmentText += "$($entry.name)=$($entry.value)`n" - } - } elseif ($OutFileFormat -eq 'pwshenv') { - foreach($entry in $deploymentOutputs.GetEnumerator()) { - $environmentText += "`$$($entry.name)=$($entry.value)`n" - } - } + $environmentText = $deploymentOutputs | ConvertTo-Json; $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) diff --git a/eng/common/TestResources/deploy-stress-test-resources.ps1 b/eng/common/TestResources/deploy-stress-test-resources.ps1 deleted file mode 100644 index 96a3ddcc8840..000000000000 --- a/eng/common/TestResources/deploy-stress-test-resources.ps1 +++ /dev/null @@ -1,40 +0,0 @@ -param ( - [Parameter()] - [string] $TemplateParametersPath -) - -$secrets = @{} -$secretsDir = "/mnt/secrets/static/*" -Get-ChildItem -Path $secretsDir | ForEach-Object { - foreach($line in Get-Content $_) { - $idx = $line.IndexOf("=") - if ($idx -gt 0) { - $key = $line.Substring(0, $idx) - $val = $line.Substring($idx + 1) - $secrets.Add($key, $val) - } - } -} - -$templateParams = Get-Content $TemplateParametersPath | ConvertFrom-json -AsHashTable -mkdir /azure -Copy-Item "/common/scripts/stress-testing/test-resources-post.ps1" -Destination "/azure/" -Copy-Item "/mnt/testresources/*" -Destination "/azure/" - -& $PSScriptRoot/New-TestResources.ps1 ` - -BaseName $env:BASE_NAME ` - -SubscriptionId $secrets.AZURE_SUBSCRIPTION_ID ` - -TenantId $secrets.AZURE_TENANT_ID ` - -ProvisionerApplicationId $secrets.AZURE_CLIENT_ID ` - -ProvisionerApplicationSecret $secrets.AZURE_CLIENT_SECRET ` - -TestApplicationId $secrets.AZURE_CLIENT_ID ` - -TestApplicationSecret $secrets.AZURE_CLIENT_SECRET ` - -TestApplicationOid $secrets.AZURE_CLIENT_OID ` - -ArmTemplateParameters $templateParams ` - -Location 'westus2' ` - -DeleteAfterHours 168 ` - -ServiceDirectory '/azure/' ` - -CI ` - -Force - -#> \ No newline at end of file diff --git a/eng/common/scripts/stress-testing/test-resources-post.ps1 b/eng/common/scripts/stress-testing/test-resources-post.ps1 deleted file mode 100644 index d71c03f48265..000000000000 --- a/eng/common/scripts/stress-testing/test-resources-post.ps1 +++ /dev/null @@ -1,17 +0,0 @@ -param ( - [Parameter()] - [hashtable] $DeploymentOutputs, - - [Parameter(ValueFromRemainingArguments = $true)] - $RemainingArguments -) - -$outputFile = "$env:ENV_FILE" - -$environmentText = '' -foreach($entry in $DeploymentOutputs.GetEnumerator()) { - $environmentText += "$($entry.name)=$($entry.value)`n" -} - -$bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) -Set-Content $outputFile -Value $bytes -AsByteStream -Force