From 75c4aabd7fc37714b9a6caa348b8e53eee64a76b Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Thu, 17 Feb 2022 16:23:20 +0530 Subject: [PATCH 1/9] Adding SKU Recommendation Command --- .../azext_datamigration/manual/_help.py | 24 ++ .../azext_datamigration/manual/_params.py | 28 +- .../azext_datamigration/manual/commands.py | 10 + .../azext_datamigration/manual/custom.py | 321 +++++------------- .../azext_datamigration/manual/helper.py | 272 +++++++++++++++ 5 files changed, 426 insertions(+), 229 deletions(-) create mode 100644 src/datamigration/azext_datamigration/manual/helper.py diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index dd59f79a434..88d64ba4089 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -24,6 +24,30 @@ az datamigration get-assessment --config-file-path "C:\\Users\\user\\document\\config.json" """ +helps['datamigration performance-data-collection'] = """ + type: command + short-summary: Collect performance data for given SQL Server instance(s). + examples: + - name: Collect performance data of a given SQL Server using connection string. + text: |- + az datamigration performance-data-collection + - name: Collect performance data of a given SQL Server using assessment config file. + text: |- + az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json" +""" + +helps['datamigration get-sku-recommendation'] = """ + type: command + short-summary: Gives SKU recommendations for Azure SQL offerings. + examples: + - name: Get SKU recommendation for given SQL Server using command line. + text: |- + az datamigration get-sku-recommendation + - name: Get SKU recommendation for given SQL Server using assessment config file. + text: |- + az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json" +""" + helps['datamigration register-integration-runtime'] = """ type: command short-summary: Register Database Migration Service on Integration Runtime diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 96d05548e6c..14325d3a6c9 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -14,11 +14,35 @@ def load_arguments(self, _): with self.argument_context('datamigration get-assessment') as c: - c.argument('connection_string', nargs='+', help='Sql Server Connection Strings') + c.argument('connection_string', nargs='+', help='SQL Server Connection Strings') c.argument('output_folder', type=str, help='Output folder to store assessment report') c.argument('config_file_path', type=str, help='Path of the ConfigFile') c.argument('overwrite', help='Enable this parameter to overwrite the existing assessment report') + + with self.argument_context('datamigration performance-data-collection') as c: + c.argument('connection_string', type=str, help='SQL Server Connection Strings') + c.argument('output_folder', type=str, help='Output folder to store performance data') + c.argument('perf_query_interval', type=int, help='Interval at which to query performance data, in seconds. (Default: 30)') + c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds. (Default: 3600)') + c.argument('number_of_interation', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. (Default: 20, Minimum: 2)') + c.argument('config_file_path', type=str, help='Path of the ConfigFile') + + with self.argument_context('datamigration get-sku-recommendation') as c: + c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') + c.argument('target_platform', type=str, help='Target platform for SKU recommendation: either AzureSqlDatabase, AzureSqlManagedInstance, AzureSqlVirtualMachine, or Any. If Any is selected, then SKU recommendations for all three target platforms will be evaluated, and the best fit will be returned. (Default: Any)') + c.argument('target_sql_instance', type=str, help='Name of the SQL instance that SKU recommendation will be targeting. (Default: outputFolder will be scanned for files created by the PerfDataCollection action, and recommendations will be provided for every instance found)') + c.argument('target_percentile', type=int, help='Percentile of data points to be used during aggregation of the performance data. Only used for baseline (non-elastic) strategy. (Default: 95)') + c.argument('scaling_factor', type=int, help='Scaling (comfort) factor used during SKU recommendation. For example, if it is determined that there is a 4 vCore CPU requirement with a scaling factor of 150%, then the true CPU requirement will be 6 vCores. (Default: 100)') + c.argument('start_time', type=str, help='UTC start time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. (Default: all data points collected will be considered)') + c.argument('end_time', type=str, help='UTC end time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. (Default: all data points collected will be considered)') + c.argument('overwrite', help='Whether or not to overwrite any existing SKU recommendation reports. (Default: true)') + c.argument('display_result', help='Whether or not to print the SKU recommendation results to the console. (Default: true)') + c.argument('elastic_strategy', help='Whether or not to use the elastic strategy for SKU recommendations based on resource usage profiling. (Default: false)') + c.argument('database_allow_list', nargs='+', help='Space separated list of names of databases to be allowed for SKU recommendation consideration while excluding all others. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') + c.argument('database_deny_list', nargs='+', help='Space separated list of names of databases to not be considered for SKU recommendation. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') + c.argument('config_file_path', type=str, help='Path of the ConfigFile') + with self.argument_context('datamigration register-integration-runtime') as c: - c.argument('auth_key', type=str, help='AuthKey of Sql Migration Service') + c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service') c.argument('ir_path', type=str, help='Path of Integration Runtime MSI') diff --git a/src/datamigration/azext_datamigration/manual/commands.py b/src/datamigration/azext_datamigration/manual/commands.py index 0077dc64bde..31f9dad7287 100644 --- a/src/datamigration/azext_datamigration/manual/commands.py +++ b/src/datamigration/azext_datamigration/manual/commands.py @@ -18,6 +18,16 @@ def load_command_table(self, _): 'datamigration get-assessment' ) as g: g.custom_command('', 'datamigration_assessment') + + with self.command_group( + 'datamigration performance-data-collection' + ) as g: + g.custom_command('', 'datamigration_performance_data_collection') + + with self.command_group( + 'datamigration get-sku-recommendation' + ) as g: + g.custom_command('', 'datamigration_get_sku_recommendation') with self.command_group( 'datamigration register-integration-runtime' diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index b339bab1663..b1a9f93809a 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -11,31 +11,14 @@ # pylint: disable=unused-argument # pylint: disable=line-too-long -import ctypes -import json +import azext_datamigration.manual.helper as helper import os -import platform import subprocess -import time -import urllib.request -from zipfile import ZipFile -from azure.cli.core.azclierror import CLIInternalError -from azure.cli.core.azclierror import FileOperationError -from azure.cli.core.azclierror import InvalidArgumentValueError from azure.cli.core.azclierror import MutuallyExclusiveArgumentError from azure.cli.core.azclierror import RequiredArgumentMissingError from azure.cli.core.azclierror import UnclassifiedUserFault -# ----------------------------------------------------------------------------------------------------------------- -# Common helper function to validate if the commands are running on Windows. -# ----------------------------------------------------------------------------------------------------------------- -def validate_os_env(): - - if not platform.system().__contains__('Windows'): - raise CLIInternalError("This command cannot be run in non-windows environment. Please run this command in Windows environment") - - # ----------------------------------------------------------------------------------------------------------------- # Assessment Command Implementation. # ----------------------------------------------------------------------------------------------------------------- @@ -46,28 +29,7 @@ def datamigration_assessment(connection_string=None, try: - validate_os_env() - - defaultOutputFolder = get_default_output_folder() - - # Assigning base folder path - baseFolder = os.path.join(defaultOutputFolder, "Downloads") - exePath = os.path.join(baseFolder, "SqlAssessment.Console.csproj", "SqlAssessment.exe") - - # Creating base folder structure - if not os.path.exists(baseFolder): - os.makedirs(baseFolder) - - testPath = os.path.exists(exePath) - - # Downloading console app zip and extracting it - if not testPath: - zipSource = "https://sqlassess.blob.core.windows.net/app/SqlAssessment.zip" - zipDestination = os.path.join(baseFolder, "SqlAssessment.zip") - - urllib.request.urlretrieve(zipSource, filename=zipDestination) - with ZipFile(zipDestination, 'r') as zipFile: - zipFile.extractall(path=baseFolder) + defaultOutputFolder, exePath = helper.console_app_setup() if connection_string is not None and config_file_path is not None: raise MutuallyExclusiveArgumentError("Both connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") @@ -78,7 +40,7 @@ def datamigration_assessment(connection_string=None, cmd += '--overwrite False' if overwrite is False else '' subprocess.call(cmd, shell=False) elif config_file_path is not None: - validate_config_file_path(config_file_path) + helper.validate_config_file_path(config_file_path, "assess") cmd = f'{exePath} --configFile "{config_file_path}"' subprocess.call(cmd, shell=False) else: @@ -93,213 +55,118 @@ def datamigration_assessment(connection_string=None, # ----------------------------------------------------------------------------------------------------------------- -# Assessment helper function to test whether the given cofig_file_path is valid and has valid action specified. +# Performance Data Collection Command Implementation. # ----------------------------------------------------------------------------------------------------------------- -def validate_config_file_path(path): - - if not os.path.exists(path): - raise InvalidArgumentValueError(f'Invalid config file path: {path}. Please provide a valid config file path.') - - # JSON file - with open(path, "r", encoding=None) as f: - configJson = json.loads(f.read()) +def datamigration_performance_data_collection(connection_string=None, + output_folder=None, + perf_query_interval=None, + static_query_interval=None, + number_of_interation=None, + config_file_path=None): + try: - if not configJson['action'].strip().lower() == "assess": - raise FileOperationError("The desired action in config file was invalid. Please use \"Assess\" for action property in config file") - except KeyError as e: - raise FileOperationError("Invalid schema of config file. Please ensure that this is a properly formatted config file.") from e - - -# ----------------------------------------------------------------------------------------------------------------- -# Assessment helper function to return the default output folder path depending on OS environment. -# ----------------------------------------------------------------------------------------------------------------- -def get_default_output_folder(): - - osPlatform = platform.system() - if osPlatform.__contains__('Linux'): - defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), ".config", "Microsoft", "SqlAssessment") - elif osPlatform.__contains__('Darwin'): - defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), "Library", "Application Support", "Microsoft", "SqlAssessment") - else: - defaultOutputPath = os.path.join(os.getenv('LOCALAPPDATA'), "Microsoft", "SqlAssessment") + defaultOutputFolder, exePath = helper.console_app_setup() - return defaultOutputPath - - -# ----------------------------------------------------------------------------------------------------------------- -# Register Sql Migration Service on IR command Implementation. -# ----------------------------------------------------------------------------------------------------------------- -def datamigration_register_ir(auth_key, - ir_path=None): - - validate_os_env() - - if not is_user_admin(): - raise UnclassifiedUserFault("Failed: You do not have Administrator rights to run this command. Please re-run this command as an Administrator!") - validate_input(auth_key) - if ir_path is not None: - install_gateway(ir_path) - - register_ir(auth_key) + if connection_string is not None and config_file_path is not None: + raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") + if connection_string is not None: + parameterList = { + "--sqlConnectionStrings" : connection_string, + "--outputFolder" : output_folder, + "--perfQueryIntervalInSec" : perf_query_interval, + "--staticQueryIntervalInSec" : static_query_interval, + "--numberOfIterations" : number_of_interation + } + cmd = f'{exePath} PerfDataCollection' + for param in parameterList: + if parameterList[param] is not None: + cmd += f' {param} "{parameterList[param]}"' + subprocess.call(cmd, shell=False) + elif config_file_path is not None: + helper.validate_config_file_path(config_file_path, "perfdatacollection") + cmd = f'{exePath} --configFile "{config_file_path}"' + subprocess.call(cmd, shell=False) + else: + raise RequiredArgumentMissingError('No valid parameter set used. Please provide any one of the these prameters: sql_connection_string, config_file_path') -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to check IR path Extension -# ----------------------------------------------------------------------------------------------------------------- -def validate_ir_extension(ir_path): + # Printing log file path + logFilePath = os.path.join(defaultOutputFolder, "Logs") + print(f"Event and Error Logs Folder Path: {logFilePath}") - if ir_path is not None: - ir_extension = os.path.splitext(ir_path)[1] - if ir_extension != ".msi": - raise InvalidArgumentValueError("Invalid Integration Runtime Extension. Please provide a valid Integration Runtime MSI path.") + except Exception as e: + raise e # ----------------------------------------------------------------------------------------------------------------- -# Helper function to check whether the command is run as admin. +# Get SKU Recommendation Command Implementation. # ----------------------------------------------------------------------------------------------------------------- -def is_user_admin(): - +def datamigration_get_sku_recommendation(output_folder=None, + target_platform=None, + target_sql_instance=None, + target_percentile=None, + scaling_factor=None, + start_time=None, + end_time=None, + overwrite=False, + display_result=False, + elastic_strategy=False, + database_allow_list=None, + database_deny_list=None, + config_file_path=None): + try: - isAdmin = os.getuid() == 0 - except AttributeError: - isAdmin = ctypes.windll.shell32.IsUserAnAdmin() != 0 - - return isAdmin + defaultOutputFolder, exePath = helper.console_app_setup() + if output_folder is not None and config_file_path is not None: + raise MutuallyExclusiveArgumentError("Both output_folder and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to validate key input. -# ----------------------------------------------------------------------------------------------------------------- -def validate_input(key): - if key == "": - raise InvalidArgumentValueError("Failed: IR Auth key is empty. Please provide a valid auth key.") - - -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to check whether SHIR is installed or not. -# ----------------------------------------------------------------------------------------------------------------- -def check_whether_gateway_installed(name): - - import winreg - # Connecting to key in registry - accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) - - # Get the path of Installed softwares - accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") - - for i in range(0, winreg.QueryInfoKey(accessKey)[0]): - installedSoftware = winreg.EnumKey(accessKey, i) - installedSoftwareKey = winreg.OpenKey(accessKey, installedSoftware) - try: - displayName = winreg.QueryValueEx(installedSoftwareKey, r"DisplayName")[0] - if name in displayName: - return True - except FileNotFoundError: - pass - - # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) - try: - diaCmdPath = get_cmd_file_path_static() - if os.path.exists(diaCmdPath): - return True + if config_file_path is not None: + helper.validate_config_file_path(config_file_path, "getskurecommendation") + cmd = f'{exePath} --configFile "{config_file_path}"' + subprocess.call(cmd, shell=False) else: - return False - except (FileNotFoundError, IndexError): - return False - - -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to install SHIR -# ----------------------------------------------------------------------------------------------------------------- -def install_gateway(path): - - if check_whether_gateway_installed("Microsoft Integration Runtime"): - print("Microsoft Integration Runtime is already installed") - return - - validate_ir_extension(path) - - if not os.path.exists(path): - raise InvalidArgumentValueError(f"Invalid Integration Runtime MSI path : {path}. Please provide a valid Integration Runtime MSI path") - - print("Start Integration Runtime installation") - - installCmd = f'msiexec.exe /i "{path}" /quiet /passive' - subprocess.call(installCmd, shell=False) - time.sleep(30) - - print("Integration Runtime installation is complete") - - -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to register Sql Migration Service on IR -# ----------------------------------------------------------------------------------------------------------------- -def register_ir(key): - print(f"Start to register IR with key: {key}") - - cmdFilePath = get_cmd_file_path() - - directoryPath = os.path.dirname(cmdFilePath) - parentDirPath = os.path.dirname(directoryPath) - - dmgCmdPath = os.path.join(directoryPath, "dmgcmd.exe") - regIRScriptPath = os.path.join(parentDirPath, "PowerShellScript", "RegisterIntegrationRuntime.ps1") - - portCmd = f'{dmgCmdPath} -EnableRemoteAccess 8060' - irCmd = f'powershell -command "& \'{regIRScriptPath}\' -gatewayKey {key}"' - - subprocess.call(portCmd, shell=False) - subprocess.call(irCmd, shell=False) - - -# ----------------------------------------------------------------------------------------------------------------- -# Helper function to get SHIR script path -# ----------------------------------------------------------------------------------------------------------------- -def get_cmd_file_path(): - - import winreg - try: - # Connecting to key in registry - accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + parameterList = { + "--outputFolder" : output_folder, + "--targetPlatform" : target_platform, + "--targetSqlInstance" : target_sql_instance, + "--scalingFactor" : scaling_factor, + "--targetPercentile" : target_percentile, + "--startTime" : start_time, + "--endTime" : end_time, + "--overwrite" : overwrite, + "--displayResult" : display_result, + "--elasticStrategy" : elastic_strategy, + "--databaseAllowList" : database_allow_list, + "--databaseDenyList" : database_deny_list + } + cmd = f'{exePath} GetSkuRecommendation' + for param in parameterList: + if parameterList[param] is not None: + cmd += f' {param} "{parameterList[param]}"' + subprocess.call(cmd, shell=False) - # Get the path of Integration Runtime - accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\DataTransfer\DataManagementGateway\ConfigurationManager") - accessValue = winreg.QueryValueEx(accessKey, r"DiacmdPath")[0] + # Printing log file path + logFilePath = os.path.join(defaultOutputFolder, "Logs") + print(f"Event and Error Logs Folder Path: {logFilePath}") - return accessValue - except FileNotFoundError: - try: - diaCmdPath = get_cmd_file_path_static() - return diaCmdPath - except FileNotFoundError as e: - raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command") from e - except IndexError as e: - raise FileOperationError("IR is not properly installed. Please re-install it and re-run this command") from e + except Exception as e: + raise e # ----------------------------------------------------------------------------------------------------------------- -# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files +# Register Sql Migration Service on IR command Implementation. # ----------------------------------------------------------------------------------------------------------------- -def get_cmd_file_path_static(): - - # Base folder is taken as Program files or Program files (x86). - baseFolderX64 = os.path.join(r"C:\Program Files", "Microsoft Integration Runtime") - baseFolderX86 = os.path.join(r"C:\Program Files (x86)", "Microsoft Integration Runtime") - if os.path.exists(baseFolderX86): - baseFolder = baseFolderX86 - else: - baseFolder = baseFolderX64 - - # Add the latest version to baseFolder path. - listDir = os.listdir(baseFolder) - listDir.sort(reverse=True) - versionFolder = os.path.join(baseFolder, listDir[0]) +def datamigration_register_ir(auth_key, + ir_path=None): - # Create diaCmd default path and check if it is valid or not. - diaCmdPath = os.path.join(versionFolder, "Shared", "diacmd.exe") + helper.validate_os_env() - if not os.path.exists(diaCmdPath): - raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") + if not helper.is_user_admin(): + raise UnclassifiedUserFault("Failed: You do not have Administrator rights to run this command. Please re-run this command as an Administrator!") + helper.validate_input(auth_key) + if ir_path is not None: + helper.install_gateway(ir_path) - return diaCmdPath + helper.register_ir(auth_key) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py new file mode 100644 index 00000000000..82a33fdba7c --- /dev/null +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -0,0 +1,272 @@ +# -------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# +# Code generated by Microsoft (R) AutoRest Code Generator. +# Changes may cause incorrect behavior and will be lost if the code is +# regenerated. +# -------------------------------------------------------------------------- +# pylint: disable=too-many-lines +# pylint: disable=unused-argument +# pylint: disable=line-too-long + +import ctypes +import json +import os +import platform +import subprocess +import time +import urllib.request +from zipfile import ZipFile +from azure.cli.core.azclierror import CLIInternalError +from azure.cli.core.azclierror import FileOperationError +from azure.cli.core.azclierror import InvalidArgumentValueError + + +# ----------------------------------------------------------------------------------------------------------------- +# Common helper function to validate if the commands are running on Windows. +# ----------------------------------------------------------------------------------------------------------------- +def validate_os_env(): + + if not platform.system().__contains__('Windows'): + raise CLIInternalError("This command cannot be run in non-windows environment. Please run this command in Windows environment") + + +# ----------------------------------------------------------------------------------------------------------------- +# Assessment helper function to test whether the given cofig_file_path is valid and has valid action specified. +# ----------------------------------------------------------------------------------------------------------------- +def validate_config_file_path(path, action): + + if not os.path.exists(path): + raise InvalidArgumentValueError(f'Invalid config file path: {path}. Please provide a valid config file path.') + + # JSON file + with open(path, "r", encoding=None) as f: + configJson = json.loads(f.read()) + try: + if not configJson['action'].strip().lower() == action: + raise FileOperationError(f"The desired action in config file was invalid. Please use \"{action}\" for action property in config file") + except KeyError as e: + raise FileOperationError("Invalid schema of config file. Please ensure that this is a properly formatted config file.") from e + + +def console_app_setup(): + + validate_os_env() + + defaultOutputFolder = get_default_output_folder() + + # Assigning base folder path + baseFolder = os.path.join(defaultOutputFolder, "Downloads") + exePath = os.path.join(baseFolder, "SqlAssessment.Console.csproj", "SqlAssessment.exe") + + # Creating base folder structure + create_dir_path(baseFolder) + # check and download console app + check_and_download_console_app(exePath, baseFolder) + + return defaultOutputFolder, exePath + +# ----------------------------------------------------------------------------------------------------------------- +# Assessment helper function to return the default output folder path depending on OS environment. +# ----------------------------------------------------------------------------------------------------------------- +def get_default_output_folder(): + + osPlatform = platform.system() + + if osPlatform.__contains__('Linux'): + defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), ".config", "Microsoft", "SqlAssessment") + elif osPlatform.__contains__('Darwin'): + defaultOutputPath = os.path.join(os.getenv('USERPROFILE'), "Library", "Application Support", "Microsoft", "SqlAssessment") + else: + defaultOutputPath = os.path.join(os.getenv('LOCALAPPDATA'), "Microsoft", "SqlAssessment") + + return defaultOutputPath + + +# ----------------------------------------------------------------------------------------------------------------- +# Assessment helper function to check if console app exists, if not download it. +# ----------------------------------------------------------------------------------------------------------------- +def check_and_download_console_app(exePath, baseFolder): + + testPath = os.path.exists(exePath) + + # Downloading console app zip and extracting it + if not testPath: + zipSource = "https://sqlassess.blob.core.windows.net/app/SqlAssessment.zip" + zipDestination = os.path.join(baseFolder, "SqlAssessment.zip") + + urllib.request.urlretrieve(zipSource, filename=zipDestination) + with ZipFile(zipDestination, 'r') as zipFile: + zipFile.extractall(path=baseFolder) + + +# ----------------------------------------------------------------------------------------------------------------- +# Assessment helper function to check if baseFolder exists, if not create it. +# ----------------------------------------------------------------------------------------------------------------- +def create_dir_path(baseFolder): + + if not os.path.exists(baseFolder): + os.makedirs(baseFolder) + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to check IR path Extension +# ----------------------------------------------------------------------------------------------------------------- +def validate_ir_extension(ir_path): + + if ir_path is not None: + ir_extension = os.path.splitext(ir_path)[1] + if ir_extension != ".msi": + raise InvalidArgumentValueError("Invalid Integration Runtime Extension. Please provide a valid Integration Runtime MSI path.") + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to check whether the command is run as admin. +# ----------------------------------------------------------------------------------------------------------------- +def is_user_admin(): + + try: + isAdmin = os.getuid() == 0 + except AttributeError: + isAdmin = ctypes.windll.shell32.IsUserAnAdmin() != 0 + + return isAdmin + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to validate key input. +# ----------------------------------------------------------------------------------------------------------------- +def validate_input(key): + if key == "": + raise InvalidArgumentValueError("Failed: IR Auth key is empty. Please provide a valid auth key.") + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to check whether SHIR is installed or not. +# ----------------------------------------------------------------------------------------------------------------- +def check_whether_gateway_installed(name): + + import winreg + # Connecting to key in registry + accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + + # Get the path of Installed softwares + accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") + + for i in range(0, winreg.QueryInfoKey(accessKey)[0]): + installedSoftware = winreg.EnumKey(accessKey, i) + installedSoftwareKey = winreg.OpenKey(accessKey, installedSoftware) + try: + displayName = winreg.QueryValueEx(installedSoftwareKey, r"DisplayName")[0] + if name in displayName: + return True + except FileNotFoundError: + pass + + # Adding this try to look for Installed IR in Program files (Assumes the IR is always installed there) + try: + diaCmdPath = get_cmd_file_path_static() + if os.path.exists(diaCmdPath): + return True + else: + return False + except (FileNotFoundError, IndexError): + return False + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to install SHIR +# ----------------------------------------------------------------------------------------------------------------- +def install_gateway(path): + + if check_whether_gateway_installed("Microsoft Integration Runtime"): + print("Microsoft Integration Runtime is already installed") + return + + validate_ir_extension(path) + + if not os.path.exists(path): + raise InvalidArgumentValueError(f"Invalid Integration Runtime MSI path : {path}. Please provide a valid Integration Runtime MSI path") + + print("Start Integration Runtime installation") + + installCmd = f'msiexec.exe /i "{path}" /quiet /passive' + subprocess.call(installCmd, shell=False) + time.sleep(30) + + print("Integration Runtime installation is complete") + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to register Sql Migration Service on IR +# ----------------------------------------------------------------------------------------------------------------- +def register_ir(key): + print(f"Start to register IR with key: {key}") + + cmdFilePath = get_cmd_file_path() + + directoryPath = os.path.dirname(cmdFilePath) + parentDirPath = os.path.dirname(directoryPath) + + dmgCmdPath = os.path.join(directoryPath, "dmgcmd.exe") + regIRScriptPath = os.path.join(parentDirPath, "PowerShellScript", "RegisterIntegrationRuntime.ps1") + + portCmd = f'{dmgCmdPath} -EnableRemoteAccess 8060' + irCmd = f'powershell -command "& \'{regIRScriptPath}\' -gatewayKey {key}"' + + subprocess.call(portCmd, shell=False) + subprocess.call(irCmd, shell=False) + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to get SHIR script path +# ----------------------------------------------------------------------------------------------------------------- +def get_cmd_file_path(): + + import winreg + try: + # Connecting to key in registry + accessRegistry = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) + + # Get the path of Integration Runtime + accessKey = winreg.OpenKey(accessRegistry, r"SOFTWARE\Microsoft\DataTransfer\DataManagementGateway\ConfigurationManager") + accessValue = winreg.QueryValueEx(accessKey, r"DiacmdPath")[0] + + return accessValue + except FileNotFoundError: + try: + diaCmdPath = get_cmd_file_path_static() + return diaCmdPath + except FileNotFoundError as e: + raise FileOperationError("Failed: No installed IR found or installed IR is not present in Program Files. Please install Integration Runtime in default location and re-run this command") from e + except IndexError as e: + raise FileOperationError("IR is not properly installed. Please re-install it and re-run this command") from e + + +# ----------------------------------------------------------------------------------------------------------------- +# Helper function to get DiaCmdPath with Static Paths. This function assumes that IR is always installed in program files +# ----------------------------------------------------------------------------------------------------------------- +def get_cmd_file_path_static(): + + # Base folder is taken as Program files or Program files (x86). + baseFolderX64 = os.path.join(r"C:\Program Files", "Microsoft Integration Runtime") + baseFolderX86 = os.path.join(r"C:\Program Files (x86)", "Microsoft Integration Runtime") + if os.path.exists(baseFolderX86): + baseFolder = baseFolderX86 + else: + baseFolder = baseFolderX64 + + # Add the latest version to baseFolder path. + listDir = os.listdir(baseFolder) + listDir.sort(reverse=True) + versionFolder = os.path.join(baseFolder, listDir[0]) + + # Create diaCmd default path and check if it is valid or not. + diaCmdPath = os.path.join(versionFolder, "Shared", "diacmd.exe") + + if not os.path.exists(diaCmdPath): + raise FileNotFoundError(f"The system cannot find the path specified: {diaCmdPath}") + + return diaCmdPath From 4e0df55933f2bf3e63d125b08e481dac3ec4e8af Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Thu, 17 Feb 2022 16:26:32 +0530 Subject: [PATCH 2/9] Adding multiserver support - currently fails --- src/datamigration/azext_datamigration/manual/_params.py | 2 +- src/datamigration/azext_datamigration/manual/custom.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 14325d3a6c9..4a264dc4f31 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -20,7 +20,7 @@ def load_arguments(self, _): c.argument('overwrite', help='Enable this parameter to overwrite the existing assessment report') with self.argument_context('datamigration performance-data-collection') as c: - c.argument('connection_string', type=str, help='SQL Server Connection Strings') + c.argument('connection_string', nargs='+', help='SQL Server Connection Strings') c.argument('output_folder', type=str, help='Output folder to store performance data') c.argument('perf_query_interval', type=int, help='Interval at which to query performance data, in seconds. (Default: 30)') c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds. (Default: 3600)') diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index b1a9f93809a..4e1e0579506 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -72,6 +72,7 @@ def datamigration_performance_data_collection(connection_string=None, raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") if connection_string is not None: + connection_string = ", ".join(f"\"{i}\"" for i in connection_string) parameterList = { "--sqlConnectionStrings" : connection_string, "--outputFolder" : output_folder, From 4636806f8ce342f358695ca942f4485404bd30c6 Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Fri, 18 Feb 2022 02:00:36 +0530 Subject: [PATCH 3/9] Adding Support for DatabaseAllowList and DatabaseDenyList --- .../azext_datamigration/manual/_params.py | 2 +- .../azext_datamigration/manual/custom.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 4a264dc4f31..a5e00a15952 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -30,7 +30,7 @@ def load_arguments(self, _): with self.argument_context('datamigration get-sku-recommendation') as c: c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') c.argument('target_platform', type=str, help='Target platform for SKU recommendation: either AzureSqlDatabase, AzureSqlManagedInstance, AzureSqlVirtualMachine, or Any. If Any is selected, then SKU recommendations for all three target platforms will be evaluated, and the best fit will be returned. (Default: Any)') - c.argument('target_sql_instance', type=str, help='Name of the SQL instance that SKU recommendation will be targeting. (Default: outputFolder will be scanned for files created by the PerfDataCollection action, and recommendations will be provided for every instance found)') + c.argument('target_sql_instance', type=str, help='Name of the SQL instance for which SKU should be recommendeded. (Default: outputFolder will be scanned for files created by the PerfDataCollection action, and recommendations will be provided for every instance found)') c.argument('target_percentile', type=int, help='Percentile of data points to be used during aggregation of the performance data. Only used for baseline (non-elastic) strategy. (Default: 95)') c.argument('scaling_factor', type=int, help='Scaling (comfort) factor used during SKU recommendation. For example, if it is determined that there is a 4 vCore CPU requirement with a scaling factor of 150%, then the true CPU requirement will be 6 vCores. (Default: 100)') c.argument('start_time', type=str, help='UTC start time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. (Default: all data points collected will be considered)') diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 4e1e0579506..cfbe8b4cb17 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -35,7 +35,7 @@ def datamigration_assessment(connection_string=None, raise MutuallyExclusiveArgumentError("Both connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") if connection_string is not None: - connection_string = ", ".join(f"\"{i}\"" for i in connection_string) + connection_string = " ".join(f"\"{i}\"" for i in connection_string) cmd = f'{exePath} Assess --sqlConnectionStrings {connection_string} ' if output_folder is None else f'{exePath} Assess --sqlConnectionStrings {connection_string} --outputFolder "{output_folder}" ' cmd += '--overwrite False' if overwrite is False else '' subprocess.call(cmd, shell=False) @@ -72,15 +72,14 @@ def datamigration_performance_data_collection(connection_string=None, raise MutuallyExclusiveArgumentError("Both sql_connection_string and config_file_path are mutually exclusive arguments. Please provide only one of these arguments.") if connection_string is not None: - connection_string = ", ".join(f"\"{i}\"" for i in connection_string) + connection_string = " ".join(f"\"{i}\"" for i in connection_string) parameterList = { - "--sqlConnectionStrings" : connection_string, "--outputFolder" : output_folder, "--perfQueryIntervalInSec" : perf_query_interval, "--staticQueryIntervalInSec" : static_query_interval, "--numberOfIterations" : number_of_interation } - cmd = f'{exePath} PerfDataCollection' + cmd = f'{exePath} PerfDataCollection --sqlConnectionStrings {connection_string}' for param in parameterList: if parameterList[param] is not None: cmd += f' {param} "{parameterList[param]}"' @@ -144,8 +143,11 @@ def datamigration_get_sku_recommendation(output_folder=None, } cmd = f'{exePath} GetSkuRecommendation' for param in parameterList: - if parameterList[param] is not None: + if parameterList[param] is not None and not param.__contains__("List"): cmd += f' {param} "{parameterList[param]}"' + elif param.__contains__("List") and parameterList[param] is not None: + parameterList[param] = " ".join(f"\"{i}\"" for i in parameterList[param]) + cmd += f' {param} {parameterList[param]}' subprocess.call(cmd, shell=False) # Printing log file path From dbfe922be2486b877091a6e7957f064235ee9acb Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Fri, 18 Feb 2022 02:25:45 +0530 Subject: [PATCH 4/9] Correcting lint and style errors --- .../azext_datamigration/manual/_help.py | 2 +- .../azext_datamigration/manual/_params.py | 6 +- .../azext_datamigration/manual/commands.py | 2 +- .../azext_datamigration/manual/custom.py | 72 +++++++++---------- .../azext_datamigration/manual/helper.py | 3 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index 88d64ba4089..ec44d2b6bf5 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -42,7 +42,7 @@ examples: - name: Get SKU recommendation for given SQL Server using command line. text: |- - az datamigration get-sku-recommendation + az datamigration get-sku-recommendation - name: Get SKU recommendation for given SQL Server using assessment config file. text: |- az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json" diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index a5e00a15952..3ef463fea65 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -9,6 +9,7 @@ # -------------------------------------------------------------------------- # pylint: disable=too-many-lines # pylint: disable=too-many-statements +# pylint: disable=line-too-long def load_arguments(self, _): @@ -18,7 +19,7 @@ def load_arguments(self, _): c.argument('output_folder', type=str, help='Output folder to store assessment report') c.argument('config_file_path', type=str, help='Path of the ConfigFile') c.argument('overwrite', help='Enable this parameter to overwrite the existing assessment report') - + with self.argument_context('datamigration performance-data-collection') as c: c.argument('connection_string', nargs='+', help='SQL Server Connection Strings') c.argument('output_folder', type=str, help='Output folder to store performance data') @@ -26,7 +27,7 @@ def load_arguments(self, _): c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds. (Default: 3600)') c.argument('number_of_interation', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. (Default: 20, Minimum: 2)') c.argument('config_file_path', type=str, help='Path of the ConfigFile') - + with self.argument_context('datamigration get-sku-recommendation') as c: c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') c.argument('target_platform', type=str, help='Target platform for SKU recommendation: either AzureSqlDatabase, AzureSqlManagedInstance, AzureSqlVirtualMachine, or Any. If Any is selected, then SKU recommendations for all three target platforms will be evaluated, and the best fit will be returned. (Default: Any)') @@ -41,7 +42,6 @@ def load_arguments(self, _): c.argument('database_allow_list', nargs='+', help='Space separated list of names of databases to be allowed for SKU recommendation consideration while excluding all others. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') c.argument('database_deny_list', nargs='+', help='Space separated list of names of databases to not be considered for SKU recommendation. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') c.argument('config_file_path', type=str, help='Path of the ConfigFile') - with self.argument_context('datamigration register-integration-runtime') as c: c.argument('auth_key', type=str, help='AuthKey of SQL Migration Service') diff --git a/src/datamigration/azext_datamigration/manual/commands.py b/src/datamigration/azext_datamigration/manual/commands.py index 31f9dad7287..6e7b828e4db 100644 --- a/src/datamigration/azext_datamigration/manual/commands.py +++ b/src/datamigration/azext_datamigration/manual/commands.py @@ -18,7 +18,7 @@ def load_command_table(self, _): 'datamigration get-assessment' ) as g: g.custom_command('', 'datamigration_assessment') - + with self.command_group( 'datamigration performance-data-collection' ) as g: diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index cfbe8b4cb17..68d79376f14 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -11,12 +11,12 @@ # pylint: disable=unused-argument # pylint: disable=line-too-long -import azext_datamigration.manual.helper as helper import os import subprocess from azure.cli.core.azclierror import MutuallyExclusiveArgumentError from azure.cli.core.azclierror import RequiredArgumentMissingError from azure.cli.core.azclierror import UnclassifiedUserFault +from azext_datamigration.manual import helper # ----------------------------------------------------------------------------------------------------------------- @@ -58,12 +58,12 @@ def datamigration_assessment(connection_string=None, # Performance Data Collection Command Implementation. # ----------------------------------------------------------------------------------------------------------------- def datamigration_performance_data_collection(connection_string=None, - output_folder=None, - perf_query_interval=None, - static_query_interval=None, - number_of_interation=None, - config_file_path=None): - + output_folder=None, + perf_query_interval=None, + static_query_interval=None, + number_of_interation=None, + config_file_path=None): + try: defaultOutputFolder, exePath = helper.console_app_setup() @@ -74,10 +74,10 @@ def datamigration_performance_data_collection(connection_string=None, if connection_string is not None: connection_string = " ".join(f"\"{i}\"" for i in connection_string) parameterList = { - "--outputFolder" : output_folder, - "--perfQueryIntervalInSec" : perf_query_interval, - "--staticQueryIntervalInSec" : static_query_interval, - "--numberOfIterations" : number_of_interation + "--outputFolder": output_folder, + "--perfQueryIntervalInSec": perf_query_interval, + "--staticQueryIntervalInSec": static_query_interval, + "--numberOfIterations": number_of_interation } cmd = f'{exePath} PerfDataCollection --sqlConnectionStrings {connection_string}' for param in parameterList: @@ -103,19 +103,19 @@ def datamigration_performance_data_collection(connection_string=None, # Get SKU Recommendation Command Implementation. # ----------------------------------------------------------------------------------------------------------------- def datamigration_get_sku_recommendation(output_folder=None, - target_platform=None, - target_sql_instance=None, - target_percentile=None, - scaling_factor=None, - start_time=None, - end_time=None, - overwrite=False, - display_result=False, - elastic_strategy=False, - database_allow_list=None, - database_deny_list=None, - config_file_path=None): - + target_platform=None, + target_sql_instance=None, + target_percentile=None, + scaling_factor=None, + start_time=None, + end_time=None, + overwrite=False, + display_result=False, + elastic_strategy=False, + database_allow_list=None, + database_deny_list=None, + config_file_path=None): + try: defaultOutputFolder, exePath = helper.console_app_setup() @@ -128,18 +128,18 @@ def datamigration_get_sku_recommendation(output_folder=None, subprocess.call(cmd, shell=False) else: parameterList = { - "--outputFolder" : output_folder, - "--targetPlatform" : target_platform, - "--targetSqlInstance" : target_sql_instance, - "--scalingFactor" : scaling_factor, - "--targetPercentile" : target_percentile, - "--startTime" : start_time, - "--endTime" : end_time, - "--overwrite" : overwrite, - "--displayResult" : display_result, - "--elasticStrategy" : elastic_strategy, - "--databaseAllowList" : database_allow_list, - "--databaseDenyList" : database_deny_list + "--outputFolder": output_folder, + "--targetPlatform": target_platform, + "--targetSqlInstance": target_sql_instance, + "--scalingFactor": scaling_factor, + "--targetPercentile": target_percentile, + "--startTime": start_time, + "--endTime": end_time, + "--overwrite": overwrite, + "--displayResult": display_result, + "--elasticStrategy": elastic_strategy, + "--databaseAllowList": database_allow_list, + "--databaseDenyList": database_deny_list } cmd = f'{exePath} GetSkuRecommendation' for param in parameterList: diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index 82a33fdba7c..8d77e449901 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -68,6 +68,7 @@ def console_app_setup(): return defaultOutputFolder, exePath + # ----------------------------------------------------------------------------------------------------------------- # Assessment helper function to return the default output folder path depending on OS environment. # ----------------------------------------------------------------------------------------------------------------- @@ -103,7 +104,7 @@ def check_and_download_console_app(exePath, baseFolder): # ----------------------------------------------------------------------------------------------------------------- -# Assessment helper function to check if baseFolder exists, if not create it. +# Assessment helper function to check if baseFolder exists, if not create it. # ----------------------------------------------------------------------------------------------------------------- def create_dir_path(baseFolder): From 83ae5aea6ede83b575497910f12f8a8b28f14a97 Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Fri, 18 Feb 2022 15:40:41 +0530 Subject: [PATCH 5/9] Adding help for perf-data-collection and sku-recommendation --- src/datamigration/azext_datamigration/manual/_help.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index ec44d2b6bf5..d8ff1e25000 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -30,7 +30,7 @@ examples: - name: Collect performance data of a given SQL Server using connection string. text: |- - az datamigration performance-data-collection + az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-interation 5 --perf-query-interval 10 --static-query-interval 60 - name: Collect performance data of a given SQL Server using assessment config file. text: |- az datamigration performance-data-collection --config-file-path "C:\\Users\\user\\document\\config.json" @@ -42,7 +42,7 @@ examples: - name: Get SKU recommendation for given SQL Server using command line. text: |- - az datamigration get-sku-recommendation + az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite - name: Get SKU recommendation for given SQL Server using assessment config file. text: |- az datamigration get-sku-recommendation --config-file-path "C:\\Users\\user\\document\\config.json" From 80f8c90800e0e91816d9a9f70127d69b86d7dccd Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Mon, 21 Feb 2022 16:02:49 +0530 Subject: [PATCH 6/9] Adding changes as per suggestions --- .../azext_datamigration/manual/_help.py | 2 +- .../azext_datamigration/manual/_params.py | 28 +++++++++---------- .../azext_datamigration/manual/commands.py | 24 ++++++---------- .../azext_datamigration/manual/custom.py | 12 ++++---- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/_help.py b/src/datamigration/azext_datamigration/manual/_help.py index d8ff1e25000..16f677597ff 100644 --- a/src/datamigration/azext_datamigration/manual/_help.py +++ b/src/datamigration/azext_datamigration/manual/_help.py @@ -38,7 +38,7 @@ helps['datamigration get-sku-recommendation'] = """ type: command - short-summary: Gives SKU recommendations for Azure SQL offerings. + short-summary: Give SKU recommendations for Azure SQL offerings. examples: - name: Get SKU recommendation for given SQL Server using command line. text: |- diff --git a/src/datamigration/azext_datamigration/manual/_params.py b/src/datamigration/azext_datamigration/manual/_params.py index 3ef463fea65..622b3cb75aa 100644 --- a/src/datamigration/azext_datamigration/manual/_params.py +++ b/src/datamigration/azext_datamigration/manual/_params.py @@ -23,24 +23,24 @@ def load_arguments(self, _): with self.argument_context('datamigration performance-data-collection') as c: c.argument('connection_string', nargs='+', help='SQL Server Connection Strings') c.argument('output_folder', type=str, help='Output folder to store performance data') - c.argument('perf_query_interval', type=int, help='Interval at which to query performance data, in seconds. (Default: 30)') - c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds. (Default: 3600)') - c.argument('number_of_interation', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. (Default: 20, Minimum: 2)') + c.argument('perf_query_interval', type=int, help='Interval at which to query performance data, in seconds.') + c.argument('static_query_interval', type=int, help='Interval at which to query and persist static configuration data, in seconds.') + c.argument('number_of_interation', type=int, help='Number of iterations of performance data collection to perform before persisting to file. For example, with default values, performance data will be persisted every 30 seconds * 20 iterations = 10 minutes. Minimum: 2.') c.argument('config_file_path', type=str, help='Path of the ConfigFile') with self.argument_context('datamigration get-sku-recommendation') as c: c.argument('output_folder', type=str, help='Output folder where performance data of the SQL Server is stored. The value here must be the same as the one used in PerfDataCollection') - c.argument('target_platform', type=str, help='Target platform for SKU recommendation: either AzureSqlDatabase, AzureSqlManagedInstance, AzureSqlVirtualMachine, or Any. If Any is selected, then SKU recommendations for all three target platforms will be evaluated, and the best fit will be returned. (Default: Any)') - c.argument('target_sql_instance', type=str, help='Name of the SQL instance for which SKU should be recommendeded. (Default: outputFolder will be scanned for files created by the PerfDataCollection action, and recommendations will be provided for every instance found)') - c.argument('target_percentile', type=int, help='Percentile of data points to be used during aggregation of the performance data. Only used for baseline (non-elastic) strategy. (Default: 95)') - c.argument('scaling_factor', type=int, help='Scaling (comfort) factor used during SKU recommendation. For example, if it is determined that there is a 4 vCore CPU requirement with a scaling factor of 150%, then the true CPU requirement will be 6 vCores. (Default: 100)') - c.argument('start_time', type=str, help='UTC start time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. (Default: all data points collected will be considered)') - c.argument('end_time', type=str, help='UTC end time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. (Default: all data points collected will be considered)') - c.argument('overwrite', help='Whether or not to overwrite any existing SKU recommendation reports. (Default: true)') - c.argument('display_result', help='Whether or not to print the SKU recommendation results to the console. (Default: true)') - c.argument('elastic_strategy', help='Whether or not to use the elastic strategy for SKU recommendations based on resource usage profiling. (Default: false)') - c.argument('database_allow_list', nargs='+', help='Space separated list of names of databases to be allowed for SKU recommendation consideration while excluding all others. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') - c.argument('database_deny_list', nargs='+', help='Space separated list of names of databases to not be considered for SKU recommendation. Only set one of the following or neither: databaseAllowList, databaseDenyList. (Default: null)') + c.argument('target_platform', type=str, help='Target platform for SKU recommendation: either AzureSqlDatabase, AzureSqlManagedInstance, AzureSqlVirtualMachine, or Any. If Any is selected, then SKU recommendations for all three target platforms will be evaluated, and the best fit will be returned.') + c.argument('target_sql_instance', type=str, help='Name of the SQL instance for which SKU should be recommendeded. Default: outputFolder will be scanned for files created by the PerfDataCollection action, and recommendations will be provided for every instance found.') + c.argument('target_percentile', type=int, help='Percentile of data points to be used during aggregation of the performance data. Only used for baseline (non-elastic) strategy.') + c.argument('scaling_factor', type=int, help='Scaling (comfort) factor used during SKU recommendation. For example, if it is determined that there is a 4 vCore CPU requirement with a scaling factor of 150%, then the true CPU requirement will be 6 vCores.') + c.argument('start_time', type=str, help='UTC start time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. Default: all data points collected will be considered.') + c.argument('end_time', type=str, help='UTC end time of performance data points to consider during aggregation, in YYYY-MM-DD HH:MM format. Only used for baseline (non-elastic) strategy. Default: all data points collected will be considered.') + c.argument('overwrite', help='Whether or not to overwrite any existing SKU recommendation reports. Enable this paramater to overwrite.') + c.argument('display_result', help='Whether or not to print the SKU recommendation results to the console. Enable this parameter to display result.') + c.argument('elastic_strategy', help='Whether or not to use the elastic strategy for SKU recommendations based on resource usage profiling. Enable this parameter to use elastic strategy.') + c.argument('database_allow_list', nargs='+', help='Space separated list of names of databases to be allowed for SKU recommendation consideration while excluding all others. Only set one of the following or neither: databaseAllowList, databaseDenyList. Default: null.') + c.argument('database_deny_list', nargs='+', help='Space separated list of names of databases to not be considered for SKU recommendation. Only set one of the following or neither: databaseAllowList, databaseDenyList. Default: null.') c.argument('config_file_path', type=str, help='Path of the ConfigFile') with self.argument_context('datamigration register-integration-runtime') as c: diff --git a/src/datamigration/azext_datamigration/manual/commands.py b/src/datamigration/azext_datamigration/manual/commands.py index 6e7b828e4db..9bdd46d17c0 100644 --- a/src/datamigration/azext_datamigration/manual/commands.py +++ b/src/datamigration/azext_datamigration/manual/commands.py @@ -14,22 +14,14 @@ def load_command_table(self, _): - with self.command_group( - 'datamigration get-assessment' - ) as g: - g.custom_command('', 'datamigration_assessment') + with self.command_group('datamigration') as g: + g.custom_command('get-assessment', 'datamigration_assessment') - with self.command_group( - 'datamigration performance-data-collection' - ) as g: - g.custom_command('', 'datamigration_performance_data_collection') + with self.command_group('datamigration') as g: + g.custom_command('performance-data-collection', 'datamigration_performance_data_collection') - with self.command_group( - 'datamigration get-sku-recommendation' - ) as g: - g.custom_command('', 'datamigration_get_sku_recommendation') + with self.command_group('datamigration') as g: + g.custom_command('get-sku-recommendation', 'datamigration_get_sku_recommendation') - with self.command_group( - 'datamigration register-integration-runtime' - ) as g: - g.custom_command('', 'datamigration_register_ir') + with self.command_group('datamigration') as g: + g.custom_command('register-integration-runtime', 'datamigration_register_ir') diff --git a/src/datamigration/azext_datamigration/manual/custom.py b/src/datamigration/azext_datamigration/manual/custom.py index 68d79376f14..e92532f18b9 100644 --- a/src/datamigration/azext_datamigration/manual/custom.py +++ b/src/datamigration/azext_datamigration/manual/custom.py @@ -59,9 +59,9 @@ def datamigration_assessment(connection_string=None, # ----------------------------------------------------------------------------------------------------------------- def datamigration_performance_data_collection(connection_string=None, output_folder=None, - perf_query_interval=None, - static_query_interval=None, - number_of_interation=None, + perf_query_interval=30, + static_query_interval=3600, + number_of_interation=20, config_file_path=None): try: @@ -103,10 +103,10 @@ def datamigration_performance_data_collection(connection_string=None, # Get SKU Recommendation Command Implementation. # ----------------------------------------------------------------------------------------------------------------- def datamigration_get_sku_recommendation(output_folder=None, - target_platform=None, + target_platform="Any", target_sql_instance=None, - target_percentile=None, - scaling_factor=None, + target_percentile=95, + scaling_factor=100, start_time=None, end_time=None, overwrite=False, From 00243a58588059f7cc92e342c4b320b7b6847aeb Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Mon, 21 Feb 2022 17:40:58 +0530 Subject: [PATCH 7/9] Nit change: config_file_path --- src/datamigration/azext_datamigration/manual/helper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/datamigration/azext_datamigration/manual/helper.py b/src/datamigration/azext_datamigration/manual/helper.py index 8d77e449901..ba953b9343b 100644 --- a/src/datamigration/azext_datamigration/manual/helper.py +++ b/src/datamigration/azext_datamigration/manual/helper.py @@ -34,7 +34,7 @@ def validate_os_env(): # ----------------------------------------------------------------------------------------------------------------- -# Assessment helper function to test whether the given cofig_file_path is valid and has valid action specified. +# Assessment helper function to test whether the given config_file_path is valid and has valid action specified. # ----------------------------------------------------------------------------------------------------------------- def validate_config_file_path(path, action): @@ -51,6 +51,9 @@ def validate_config_file_path(path, action): raise FileOperationError("Invalid schema of config file. Please ensure that this is a properly formatted config file.") from e +# ----------------------------------------------------------------------------------------------------------------- +# Assessment helper function to do console app setup (mkdir, download and extract) +# ----------------------------------------------------------------------------------------------------------------- def console_app_setup(): validate_os_env() From 2f2cd5e64f17132336571dbe9f22896601ed068f Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Mon, 21 Feb 2022 17:46:26 +0530 Subject: [PATCH 8/9] Adding custom cmds to common group --- src/datamigration/azext_datamigration/manual/commands.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/datamigration/azext_datamigration/manual/commands.py b/src/datamigration/azext_datamigration/manual/commands.py index 9bdd46d17c0..97218d5eeaa 100644 --- a/src/datamigration/azext_datamigration/manual/commands.py +++ b/src/datamigration/azext_datamigration/manual/commands.py @@ -16,12 +16,6 @@ def load_command_table(self, _): with self.command_group('datamigration') as g: g.custom_command('get-assessment', 'datamigration_assessment') - - with self.command_group('datamigration') as g: g.custom_command('performance-data-collection', 'datamigration_performance_data_collection') - - with self.command_group('datamigration') as g: g.custom_command('get-sku-recommendation', 'datamigration_get_sku_recommendation') - - with self.command_group('datamigration') as g: g.custom_command('register-integration-runtime', 'datamigration_register_ir') From 4b99216715cf289138c5e0d20d3a122b7bad991c Mon Sep 17 00:00:00 2001 From: Thakur Ashutosh Suman Date: Wed, 23 Feb 2022 13:25:38 +0530 Subject: [PATCH 9/9] Updating version of extension --- src/datamigration/HISTORY.rst | 6 ++++++ src/datamigration/README.md | 10 ++++++++++ src/datamigration/setup.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/datamigration/HISTORY.rst b/src/datamigration/HISTORY.rst index 1c139576ba0..fc54d120109 100644 --- a/src/datamigration/HISTORY.rst +++ b/src/datamigration/HISTORY.rst @@ -3,6 +3,12 @@ Release History =============== +0.2.0 +++++++ +* Bug fix for Multiple connection strings in az datamigration get-assessment command. +* [NEW COMMAND] az datamigration performance-data-collection - Collect performance data for given SQL Server instance(s). +* [NEW COMMAND] az datamigration get-sku-recommendation - Give SKU recommendations for Azure SQL offerings. + 0.1.0 ++++++ * Initial release. diff --git a/src/datamigration/README.md b/src/datamigration/README.md index e38e6c6d171..13e2e50cc74 100644 --- a/src/datamigration/README.md +++ b/src/datamigration/README.md @@ -19,6 +19,16 @@ az datamigration get-assessment --connection-string "Data Source=LabServer.datab az datamigration register-integration-runtime --auth-key "IR@00000-0000000-000000-aaaaa-bbbb-cccc" ``` +##### Performance-data-collection ##### +``` +az datamigration performance-data-collection --connection-string "Data Source=LabServer.database.net;Initial Catalog=master;Integrated Security=False;User Id=User;Password=password" --output-folder "C:\\PerfCollectionOutput" --number-of-interation 5 --perf-query-interval 10 --static-query-interval 60 +``` + +##### Get-sku-recommendation ##### +``` +az datamigration get-sku-recommendation --output-folder "C:\\PerfCollectionOutput" --database-allow-list AdventureWorks, AdventureWorks2 --display-result --overwrite +``` + #### datamigration sql-managed-instance #### ##### Create (Backup source Fileshare) ##### ``` diff --git a/src/datamigration/setup.py b/src/datamigration/setup.py index ae27816d3a2..175787154d2 100644 --- a/src/datamigration/setup.py +++ b/src/datamigration/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages # HISTORY.rst entry. -VERSION = '0.1.0' +VERSION = '0.2.0' try: from azext_datamigration.manual.version import VERSION except ImportError: