diff --git a/ambari-admin/src/main/resources/ui/admin-web/bower.json b/ambari-admin/src/main/resources/ui/admin-web/bower.json index fc63dda2342..5bbada910e0 100644 --- a/ambari-admin/src/main/resources/ui/admin-web/bower.json +++ b/ambari-admin/src/main/resources/ui/admin-web/bower.json @@ -1,26 +1,26 @@ { - "name": "adminconsole", - "private": true, - "dependencies": { - "bootstrap": "3.1.1", - "angular": "1.5.11", - "angular-route": "1.5.11", - "angular-bootstrap": "0.11.0", - "underscore": "1.7.0", - "restangular": "1.4.0", - "angular-bootstrap-toggle-switch": "0.5.1", - "angular-animate": "1.5.11", - "angular-translate": "2.2.0", - "font-awesome": "4.2.0" - }, - "devDependencies": { - "angular-mocks": "1.5.11", - "commonjs": "0.2.0", - "chai": "1.8.0", - "mocha": "1.14.0", - "sinon": "1.10.3" - }, - "resolutions": { - "angular": "1.5.11" - } + "name": "adminconsole", + "private": true, + "dependencies": { + "bootstrap": "3.3.7", + "angular": "1.5.11", + "angular-route": "1.5.11", + "angular-bootstrap": "0.11.0", + "underscore": "1.7.0", + "restangular": "1.4.0", + "angular-bootstrap-toggle-switch": "0.5.1", + "angular-animate": "1.5.11", + "angular-translate": "2.2.0", + "font-awesome": "4.2.0" + }, + "devDependencies": { + "angular-mocks": "1.5.11", + "commonjs": "0.2.0", + "chai": "1.8.0", + "mocha": "1.14.0", + "sinon": "1.10.3" + }, + "resolutions": { + "angular": "1.5.11" + } } diff --git a/ambari-agent/conf/unix/ambari-agent b/ambari-agent/conf/unix/ambari-agent index 816800248fc..0254484d229 100755 --- a/ambari-agent/conf/unix/ambari-agent +++ b/ambari-agent/conf/unix/ambari-agent @@ -62,7 +62,7 @@ export PATH=/usr/sbin:/sbin:/usr/lib/ambari-server/*:$PATH export AMBARI_CONF_DIR=$HOME_DIR/etc/ambari-server/conf:$PATH # Because Ambari rpm unpacks modules here on all systems -export PYTHONPATH=/usr/lib/ambari-agent/lib:${PYTHONPATH:-} +export PYTHONPATH=/usr/lib/ambari-agent/lib:/usr/lib/mpack-instance-manager:${PYTHONPATH:-} export AMBARI_PID_DIR=`get_agent_property piddir` export AMBARI_PID_DIR=`valid_path "${AMBARI_PID_DIR:?}"` diff --git a/ambari-agent/conf/unix/install-helper.sh b/ambari-agent/conf/unix/install-helper.sh index e305866130a..7c4de80bd57 100644 --- a/ambari-agent/conf/unix/install-helper.sh +++ b/ambari-agent/conf/unix/install-helper.sh @@ -117,7 +117,7 @@ do_install(){ fi if [ -f "$AMBARI_ENV_RPMSAVE" ] ; then - PYTHON_PATH_LINE='export PYTHONPATH=/usr/lib/ambari-agent/lib:$PYTHONPATH' + PYTHON_PATH_LINE='export PYTHONPATH=/usr/lib/ambari-agent/lib:/usr/lib/mpack-instance-manager:$PYTHONPATH' grep "^$PYTHON_PATH_LINE\$" "$AMBARI_ENV_RPMSAVE" > /dev/null if [ $? -ne 0 ] ; then echo -e "\n$PYTHON_PATH_LINE" >> $AMBARI_ENV_RPMSAVE diff --git a/ambari-common/src/main/python/ambari_commons/constants.py b/ambari-common/src/main/python/ambari_commons/constants.py index b59ad531c47..ff77a02032d 100644 --- a/ambari-common/src/main/python/ambari_commons/constants.py +++ b/ambari-common/src/main/python/ambari_commons/constants.py @@ -21,7 +21,7 @@ AMBARI_SUDO_BINARY = "ambari-sudo.sh" UPGRADE_TYPE_ROLLING = "rolling" -UPGRADE_TYPE_NON_ROLLING = "nonrolling" +UPGRADE_TYPE_EXPRESS = "express" UPGRADE_TYPE_HOST_ORDERED = "host_ordered" diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py index 7faa33fcad1..8a7e57779c2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -21,6 +21,7 @@ from resource_management.core.exceptions import Fail from resource_management.core.logger import Logger from resource_management.libraries.resources.repository import Repository +from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value from resource_management.libraries.functions.is_empty import is_empty import ambari_simplejson as json @@ -36,8 +37,8 @@ def __init__(self, config, tags_to_skip): # repo templates repo_file = config['repositoryFile'] - repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] - repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] + repo_rhel_suse = get_cluster_setting_value('repo_suse_rhel_template') + repo_ubuntu = get_cluster_setting_value('repo_ubuntu_template') if is_empty(repo_file): return diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/settings.py b/ambari-common/src/main/python/resource_management/libraries/functions/settings.py index 42be6d6a072..577e5babcd7 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/settings.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/settings.py @@ -55,7 +55,7 @@ def get_setting_type_entries(setting_type, setting_names=None): return None if setting_names is None: # Return all settings - return settings + setting_names = settings.keys() if not isinstance(setting_names, (set, frozenset, tuple, list)): Logger.error("'setting_names' type expected to be either a : set, frozenset, tuple, or list. " @@ -66,7 +66,7 @@ def get_setting_type_entries(setting_type, setting_names=None): Logger.error("Passed-in settings set is EMPTY") return None else: - result = dict((setting, settings[setting]) for setting in setting_names if setting in settings) + result = dict((setting, convert_value(settings[setting])) for setting in setting_names if setting in settings) if not result: Logger.error("Passed-in setting(s) in set not present.") return {} @@ -100,7 +100,7 @@ def get_setting_value(setting_type, setting_name): Logger.info("Couldn't retrieve '"+setting_type+"'.") return None - return settings.get(setting_name) + return convert_value(settings.get(setting_name)) def is_setting_type_supported(setting_type): @@ -110,3 +110,16 @@ def is_setting_type_supported(setting_type): :return: True or False """ return setting_type in (STACK_SETTINGS_TYPE, CLUSTER_SETTINGS_TYPE) + + +def convert_value(value): + """ + Converts some values for easier consumption. + Currently only strings corresponding to true/false are converted to actual boolean values. + """ + + if value == "true": + return True + if value == "false": + return False + return value diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py index 3e7331009bd..92a8383acbf 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py @@ -83,7 +83,7 @@ def check_stack_feature(stack_feature, stack_version): return False return True else: - raise Fail("Stack features not defined by stack") + Logger.warning("Stack features not defined by stack") return False diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py index 6678588a1a8..a3f086d3138 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py @@ -129,6 +129,9 @@ def is_package_supported(package, supported_packages = None): """ if supported_packages is None: supported_packages = get_supported_packages() + if supported_packages is None: + Logger.warning('No stack tool available, all packages are assumed to be supported by this stack') + return True if package in supported_packages: return True @@ -142,6 +145,9 @@ def get_supported_packages(): :return: and array of packages support by """ stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) + if not os.path.exists(stack_selector_path): + Logger.warning('This stack does not have a stack select tool') + return command = (STACK_SELECT_PREFIX, stack_selector_path, "packages") code, stdout = shell.call(command, sudo = True, quiet = True) diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 1faaf51f250..7813efc582c 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -32,7 +32,7 @@ from optparse import OptionParser import resource_management from ambari_commons import OSCheck, OSConst -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS from ambari_commons.constants import UPGRADE_TYPE_ROLLING from ambari_commons.constants import UPGRADE_TYPE_HOST_ORDERED from ambari_commons.network import reconfigure_urllib2_opener @@ -1130,8 +1130,8 @@ def get_upgrade_type(upgrade_type_command_param): upgrade_type = None if upgrade_type_command_param.lower() == "rolling_upgrade": upgrade_type = UPGRADE_TYPE_ROLLING - elif upgrade_type_command_param.lower() == "nonrolling_upgrade": - upgrade_type = UPGRADE_TYPE_NON_ROLLING + elif upgrade_type_command_param.lower() == "express_upgrade": + upgrade_type = UPGRADE_TYPE_EXPRESS elif upgrade_type_command_param.lower() == "host_ordered_upgrade": upgrade_type = UPGRADE_TYPE_HOST_ORDERED diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml deleted file mode 100644 index f345d124519..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - - - - - - - - - - zookeeper-newconfig - - - - - - - - - - - hdfs-site - - - - hdfs-site - myproperty - mynewvalue - - - hdfs-newconfig - - - - - - - - - - - core-site - - - - - important-key - - - - - - - - - - - - hive-site - hive.server2.thrift.port - 10010 - - - hive-site - hive.server2.http.port - 10011 - - - - - - hive-site - hive.server2.thrift.port - 10010 - - - hive-site - hive.server2.http.port - 10011 - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml deleted file mode 100644 index 0e6d9148d6b..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.6 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - ZOOKEEPER_SERVER - - - - - - - - - - - this is pre - - - ls - - - ls - - - - - - - ls - - - this is pre - - - this is pre - - - ls - - - ls - - - - ls - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml deleted file mode 100644 index 76e42d77889..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.5 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - UPGRADE - - HDFS - YARN - - - - - - - - actionexecute - - - - - - - DOWNGRADE - - this is downgrade message - - - - - - - actionexecute - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml deleted file mode 100644 index a44759e68b3..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml +++ /dev/null @@ -1,974 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0.1 - NON_ROLLING - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - true - false - - - - Before continuing, please stop all YARN queues. If yarn-site's yarn.resourcemanager.work-preserving-recovery.enabled is set to true, then you can skip this step since the clients will retry on their own. - - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - UPGRADE - true - false - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - KAFKA_BROKER - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - UPGRADE - true - false - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - DOWNGRADE - - - - foo_function - - - - - - - - - - - - - - UPGRADE - true - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - true - false - - - false - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - UPGRADE - - - - - wait_for_safemode_off - - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - - - - - false - true - - - TEZ_CLIENT - - - - PIG - - - - SQOOP - - - - - true - UPGRADE - - HDFS - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - - UPGRADE - true - false - - - - upgrade_oozie_database_and_sharelib - - - - - - - DOWNGRADE - true - - - - create_sharelib - - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - - true - false - - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - upgrade_oozie_database_and_sharelib - - - - - - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - delete_storm_zookeeper_data - - - - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - delete_storm_zookeeper_data - - - - - delete_storm_local_data - - - - - - - - - - - - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - delete_storm_local_data - - - - - - - - - - - - - delete_storm_local_data - - - - - - - - - - - - - delete_storm_local_data - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml deleted file mode 100644 index b6c49e9e2ee..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml +++ /dev/null @@ -1,65 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - - Foo - - - - - Back stuff up. - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml deleted file mode 100644 index 4f71b4ae8fc..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml +++ /dev/null @@ -1,211 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - Foo - - - - - Back stuff up. - - - - - ls - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - ls - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - su - {hdfs-user} -c 'dosomething' - - - - {{direction.verb.proper}} your database - - - - - - - - ls - - - - - - - - - - - Manual Downgrade - - - - - - - - - - ls - - - - - - - ls - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml deleted file mode 100644 index 85736e1096a..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml +++ /dev/null @@ -1,203 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - global-value-1 - - ^([^:]*dfs|wasb|ecs):.* - - - ^([^:]*dfs|wasb|ecs):.* - - - - - - - - - Foo - - - - - ls - - - - - Foo - - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - false - - - - - HDFS - HBASE - YARN - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests - - - - - - HDFS - YARN - - - - - - - - Please confirm you are ready to finalize - - - - - ls - - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - su - {hdfs-user} -c 'dosomething' - - - - Update your database - - - - - - - - ls - - - - - - - - - - - Manual Downgrade - - - - - - - - - ls - - - - - - - ls - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml deleted file mode 100644 index f7dd720262c..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml +++ /dev/null @@ -1,190 +0,0 @@ - - - - - - 2.2.*.* - HDP-2.2.3 - NON_ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - true - UPGRADE - - - Before continuing, please stop all YARN queues. - - - - - - UPGRADE - true - false - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - - - - - prepare_non_rolling_upgrade - - - - - - UPGRADE - true - false - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - restore_snapshot - - - - - - - - - - - - - - true - - - - actionexecute - - - - - - - - - false - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - - 2 - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - DATANODE - HDFS_CLIENT - - - - - false - true - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - NODEMANAGER - YARN_CLIENT - - - - - true - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - \ No newline at end of file diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml deleted file mode 100644 index bfd994ee11f..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml +++ /dev/null @@ -1,213 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.4 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - ls - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - ls - - - - - - - - - - - ls - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - su - {hdfs-user} -c 'dosomething' - - - - {{direction.verb.proper}} your database - - - - - - - - ls - - - - - - - - - - - Manual Downgrade - - - - - - - - - - ls - - - - - - - ls - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml deleted file mode 100644 index 90d64b40393..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - - - - - - - - - - - - - - - - - - hdfs-site - - - - - - - - - - - core-site - - - - - important-key - - - - - - - - - - - - hive-site - hive.server2.thrift.port - 10010 - - - hive-site - hive.server2.http.port - 10011 - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml deleted file mode 100644 index 5d41754a93b..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml +++ /dev/null @@ -1,221 +0,0 @@ - - - - 2.2.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - ls - - - - - Goo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - actionexecute - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - su - {hdfs-user} -c 'dosomething' - - - hdfs-site - - - - {{direction.verb.proper}} your database - - - - - - - - ls - - - - - - - - - - - Manual Downgrade - - - - - - - - - - ls - - - - - - - ls - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - hive-site - hive.server2.thrift.port - 10010 - - - hive-site - hive.server2.http.port - 10011 - - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml deleted file mode 100644 index fb03cb36e75..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.1 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - global-value-1 - - ^([^:]*dfs|wasb|ecs):.* - - - ^([^:]*dfs|wasb|ecs):.* - - - - - - - - - Foo - - - - - ls - - - - - Foo - - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - false - - - - - HDFS - YARN - - - AMBARI_METRICS - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests - - - - - - HDFS - YARN - - - AMBARI_METRICSS - - - - - - - - Please confirm you are ready to finalize - - - - - ls - - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - su - {hdfs-user} -c 'dosomething' - - - - Update your database - - - - - - - - ls - - - - - - - - - - - Manual Downgrade - - - - - - - - - ls - - - - - - - ls - - - - - - diff --git a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml b/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml deleted file mode 100644 index b2c4b9398c9..00000000000 --- a/ambari-funtest/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml +++ /dev/null @@ -1,75 +0,0 @@ - - - - 2.2.* - true - true - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - true - false - - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - diff --git a/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-container.service.ts b/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-container.service.ts index cd77bd6808f..7664b3dbf1b 100644 --- a/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-container.service.ts +++ b/ambari-logsearch/ambari-logsearch-web/src/app/services/logs-container.service.ts @@ -50,7 +50,6 @@ import { import {ListItem} from '@app/classes/list-item'; import {HomogeneousObject, LogLevelObject} from '@app/classes/object'; import {LogsType, ScrollType, SortingType} from '@app/classes/string'; - import {Tab} from '@app/classes/models/tab'; import {LogField} from '@app/classes/models/log-field'; import {AuditLog} from '@app/classes/models/audit-log'; @@ -1050,6 +1049,7 @@ export class LogsContainerService { isFilterConditionDisplayed(key: string): boolean { return this.logsTypeMap[this.activeLogsType].listFilters.indexOf(key) > -1 + && Boolean(this.filtersForm.controls[key]); } updateSelectedColumns(columnNames: string[], logsType: string): void { diff --git a/ambari-server/src/main/assemblies/server.xml b/ambari-server/src/main/assemblies/server.xml index d31b77353eb..cc6c82b992d 100644 --- a/ambari-server/src/main/assemblies/server.xml +++ b/ambari-server/src/main/assemblies/server.xml @@ -314,6 +314,11 @@ conf/unix/ca.config /var/lib/ambari-server/keys + + 755 + target/classes/cluster-settings.xml + /var/lib/ambari-server/resources + 755 target/classes/Ambari-DDL-Postgres-CREATE.sql diff --git a/ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java b/ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java index 35c3c2fe419..94be1774267 100644 --- a/ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java +++ b/ambari-server/src/main/java/org/apache/ambari/annotations/ExperimentalFeature.java @@ -45,5 +45,11 @@ public enum ExperimentalFeature { /** * For code that is for multi-service */ - MULTI_SERVICE + MULTI_SERVICE, + + /** + * For code that is for upgrading Mpacks. Use this to mark code that may ultimately + * be removed. + */ + MPACK_UPGRADES } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/ReadHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/ReadHandler.java index 077c1cc0a0f..0e307b23825 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/ReadHandler.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/ReadHandler.java @@ -121,4 +121,5 @@ private void addFieldsToQuery(Request request, Query query) { query.addProperty(propertyId, entry.getValue()); } } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/HostSummaryRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/HostSummaryRenderer.java new file mode 100644 index 00000000000..3d6b843c082 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/HostSummaryRenderer.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.api.query.render; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.ambari.server.api.services.Result; +import org.apache.ambari.server.api.services.ResultImpl; +import org.apache.ambari.server.api.util.TreeNode; +import org.apache.ambari.server.controller.internal.HostResourceProvider; +import org.apache.ambari.server.controller.internal.ResourceImpl; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.commons.lang.StringUtils; + +/** + * HostSummaryRenderere + * + * This renderer is to summarize the properties of all hosts within a cluster and returns + * a list of proerpties summary information: i.e + * + * summary:[ + * { + * "operating_systems" : { + * "centos6" : 5, + * "centos7" : 10 + * } + * } + * ] + * + */ +public class HostSummaryRenderer extends DefaultRenderer { + + // A list of host properties to be summarized + enum HostSummaryProperties { + OPERATINGSYSTEMS("operating_systems"); + + private final String property; + HostSummaryProperties(String property) { + this.property = property; + } + public String getProperty() { + return property; + } + } + + @Override + public Result finalizeResult(Result queryResult) { + TreeNode queryResultTree = queryResult.getResultTree(); + // Iterate over all returned flattened hosts and build the summary info + List summary = new ArrayList<>(); + // Build all summary info + buildFinalizedSummary(queryResultTree, summary); + // Create finalized result + return buildFinalizedResult(summary); + } + + private void buildFinalizedSummary(TreeNode queryResultTree, List summary) { + // Build osSummary info at this time + Map> osSummary = new HashMap<>(); + summary.add(osSummary); + Map osTypeCount = new HashMap<>(); + osSummary.put(HostSummaryProperties.OPERATINGSYSTEMS.getProperty(), osTypeCount); + for (TreeNode node : queryResultTree.getChildren()) { + Resource resource = node.getObject(); + String osType = (String) resource.getPropertyValue(HostResourceProvider.HOST_OS_TYPE_PROPERTY_ID); + if (StringUtils.isNotBlank(osType)) { + osTypeCount.put(osType, osTypeCount.getOrDefault(osTypeCount, 0) + 1); + } + } + } + + private Result buildFinalizedResult(List summary) { + Result result = new ResultImpl(true); + Resource resource = new ResourceImpl(Resource.Type.Host); + TreeNode summaryTree = result.getResultTree(); + summaryTree.addChild(resource, HostResourceProvider.SUMMARY_PROPERTY_ID); + resource.setProperty(HostResourceProvider.SUMMARY_PROPERTY_ID, summary); + return result; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/DetachedHostResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/DetachedHostResourceDefinition.java index 1cc42118c6d..1d19ff190a7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/DetachedHostResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/DetachedHostResourceDefinition.java @@ -19,6 +19,9 @@ package org.apache.ambari.server.api.resources; +import org.apache.ambari.server.api.query.render.HostSummaryRenderer; +import org.apache.ambari.server.api.query.render.Renderer; +import org.apache.ambari.server.controller.internal.HostResourceProvider; import org.apache.ambari.server.controller.spi.Resource; @@ -40,4 +43,13 @@ public String getPluralName() { public String getSingularName() { return "host"; } + + @Override + public Renderer getRenderer(String name) { + if (HostResourceProvider.SUMMARY_PROPERTY_ID.equals(name)) { + return new HostSummaryRenderer(); + } + return super.getRenderer(name); + } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java index d22b5f67479..d8c92afde3e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java @@ -24,6 +24,9 @@ import java.util.HashSet; import java.util.Set; +import org.apache.ambari.server.api.query.render.HostSummaryRenderer; +import org.apache.ambari.server.api.query.render.Renderer; +import org.apache.ambari.server.controller.internal.HostResourceProvider; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; @@ -54,6 +57,14 @@ public Collection getDeleteDirectives() { return Collections.singleton(Request.DIRECTIVE_DRY_RUN); } + @Override + public Renderer getRenderer(String name) { + if (HostResourceProvider.SUMMARY_PROPERTY_ID.equals(name)) { + return new HostSummaryRenderer(); + } + return super.getRenderer(name); + } + @Override public Set getSubResourceDefinitions() { Set subs = new HashSet<>(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/MpackResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/MpackResourceDefinition.java index bd9938c86df..4fbe5be916e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/MpackResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/MpackResourceDefinition.java @@ -85,7 +85,7 @@ public void process(Request request, TreeNode resultNode, String href) Map mpackInfo = mapInfo.get("MpackInfo"); int idx = href.indexOf("stacks/"); - Long mpackId = (Long)mpackInfo.get("mpack_id"); + Long mpackId = (Long)mpackInfo.get("id"); href = href.substring(0, idx) + "mpacks/" + mpackId; resultNode.setProperty("href", href); } else { @@ -95,13 +95,13 @@ public void process(Request request, TreeNode resultNode, String href) } /*** - * Post processing to change the name of the result node to current_mpack + * Post processing to change the name of the result node to mpack */ private class MpackPostProcessor implements PostProcessor { @Override public void process(Request request, TreeNode resultNode, String href) { if (href.contains("/stacks/")) { - resultNode.setName("current_mpack"); + resultNode.setName("mpack"); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackVersionResourceDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackVersionResourceDefinition.java index d621210634c..0e2026ab454 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackVersionResourceDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackVersionResourceDefinition.java @@ -92,14 +92,13 @@ public void process(Request request, TreeNode resultNode, String href) } /*** - * Post processing to change the name of the result node to current_mpack + * Post processing to change the name of the result node to stack */ private class StackVersionPostProcessor implements PostProcessor { @Override public void process(Request request, TreeNode resultNode, String href) { if (href.contains("/mpacks/")) { resultNode.setName("stack"); - } } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java index 58d1d1d0335..aa9089ec785 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java @@ -28,6 +28,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; @@ -133,7 +134,8 @@ public Response getHost(String body, @Context HttpHeaders headers, @Context UriI @ApiResponse(code = HttpStatus.SC_FORBIDDEN, message = MSG_PERMISSION_DENIED), @ApiResponse(code = HttpStatus.SC_INTERNAL_SERVER_ERROR, message = MSG_SERVER_ERROR), }) - public Response getHosts(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + public Response getHosts(String body, @Context HttpHeaders headers, @Context UriInfo ui, + @ApiParam(value = "summary", required = false) @QueryParam("format") String format) { return handleRequest(headers, body, ui, Request.Type.GET, createHostResource(m_clusterName, null)); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/MpacksService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/MpacksService.java index cf71af3fafa..ba76fe1748d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/MpacksService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/MpacksService.java @@ -73,7 +73,7 @@ public MpacksService() { response = MpackResponseWrapper.class, responseContainer = RESPONSE_CONTAINER_LIST) @ApiImplicitParams({ @ApiImplicitParam(name = QUERY_FIELDS, value = QUERY_FILTER_DESCRIPTION, dataType = DATA_TYPE_STRING, - paramType = PARAM_TYPE_QUERY, defaultValue = MpackResourceProvider.MPACK_ID), + paramType = PARAM_TYPE_QUERY, defaultValue = MpackResourceProvider.MPACK_RESOURCE_ID), @ApiImplicitParam(name = QUERY_SORT, value = QUERY_SORT_DESCRIPTION, dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY), @ApiImplicitParam(name = QUERY_PAGE_SIZE, value = QUERY_PAGE_SIZE_DESCRIPTION, defaultValue = DEFAULT_PAGE_SIZE, @@ -123,13 +123,13 @@ public Response createMpacks(String body, @Context HttpHeaders headers, @Context } /*** - * Handles: GET /mpacks/{mpack_id} - * Return a specific mpack given an mpack_id + * Handles: GET /mpacks/{id} + * Return a specific mpack given an id * * @param */ @GET - @Path("{mpack_id}") + @Path("{id}") @Produces(MediaType.TEXT_PLAIN) @ApiOperation(value = "Returns information about a specific mpack that is registered with this Ambari instance", response = MpackResponseWrapper.class) @@ -145,14 +145,14 @@ public Response createMpacks(String body, @Context HttpHeaders headers, @Context @ApiResponse(code = HttpStatus.SC_INTERNAL_SERVER_ERROR, message = MSG_SERVER_ERROR), }) public Response getMpack(String body, @Context HttpHeaders headers, @Context UriInfo ui, - @PathParam("mpack_id") String mpackId) { + @PathParam("id") String id) { return handleRequest(headers, body, ui, Request.Type.GET, - createMpackResource(mpackId)); + createMpackResource(id)); } @DELETE - @Path("{mpack_id}") + @Path("{id}") @Produces(MediaType.TEXT_PLAIN) @ApiOperation(value = "Deletes a selected management pack") @ApiImplicitParams({ @@ -166,19 +166,19 @@ public Response getMpack(String body, @Context HttpHeaders headers, @Context Uri @ApiResponse(code = HttpStatus.SC_FORBIDDEN, message = MSG_PERMISSION_DENIED), @ApiResponse(code = HttpStatus.SC_INTERNAL_SERVER_ERROR, message = MSG_SERVER_ERROR), }) - public Response deleteMpack(String body, @Context HttpHeaders headers, @Context UriInfo ui, @PathParam("mpack_id") String mpackId) { + public Response deleteMpack(String body, @Context HttpHeaders headers, @Context UriInfo ui, @PathParam("id") String id) { return handleRequest(headers, body, ui, Request.Type.DELETE, - createMpackResource(mpackId)); + createMpackResource(id)); } /** * Create an mpack resource instance - * @param mpackId + * @param id * @return ResourceInstance */ - private ResourceInstance createMpackResource(String mpackId) { + private ResourceInstance createMpackResource(String id) { return createResource(Resource.Type.Mpack, - Collections.singletonMap(Resource.Type.Mpack, mpackId)); + Collections.singletonMap(Resource.Type.Mpack, id)); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootClusterSettingService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootClusterSettingService.java index bd2fb912cf6..d8a37853791 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootClusterSettingService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/RootClusterSettingService.java @@ -70,7 +70,7 @@ public RootClusterSettingService() { response = ReadOnlyConfigurationResponse.ReadOnlyConfigurationResponseSwagger.class, responseContainer = RESPONSE_CONTAINER_LIST) @ApiImplicitParams({ @ApiImplicitParam(name = QUERY_FIELDS, value = QUERY_FILTER_DESCRIPTION, dataType = DATA_TYPE_STRING, - paramType = PARAM_TYPE_QUERY, defaultValue = MpackResourceProvider.MPACK_ID), + paramType = PARAM_TYPE_QUERY, defaultValue = MpackResourceProvider.MPACK_RESOURCE_ID), @ApiImplicitParam(name = QUERY_SORT, value = QUERY_SORT_DESCRIPTION, dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY), @ApiImplicitParam(name = QUERY_PAGE_SIZE, value = QUERY_PAGE_SIZE_DESCRIPTION, defaultValue = DEFAULT_PAGE_SIZE, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/AutoStartDisabledCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/AutoStartDisabledCheck.java index 6b0698e5b2e..a0d990682eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/AutoStartDisabledCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/AutoStartDisabledCheck.java @@ -36,7 +36,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.CONFIGURATION_WARNING, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class AutoStartDisabledCheck extends AbstractCheckDescriptor { static final String CLUSTER_ENV_TYPE = "cluster-env"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java index f26202d6c1a..8ef02db7cf9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ClientRetryPropertyCheck.java @@ -41,7 +41,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.CLIENT_RETRY_PROPERTY, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ClientRetryPropertyCheck extends AbstractCheckDescriptor { static final String HDFS_CLIENT_RETRY_DISABLED_KEY = "hdfs.client.retry.enabled.key"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsExistInRepoCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsExistInRepoCheck.java index 3ec326821c7..133eea24137 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsExistInRepoCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsExistInRepoCheck.java @@ -48,7 +48,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.TOPOLOGY, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ComponentsExistInRepoCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java index 5f486fd18d8..f960b67cd56 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ComponentsInstallationCheck.java @@ -47,7 +47,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.LIVELINESS, order = 2.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ComponentsInstallationCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ConfigurationMergeCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ConfigurationMergeCheck.java index 75730d8d7ee..d886d4f6789 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ConfigurationMergeCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ConfigurationMergeCheck.java @@ -41,7 +41,7 @@ @Singleton @UpgradeCheck( order = 99.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ConfigurationMergeCheck extends AbstractCheckDescriptor { @Inject diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheck.java index b103bcaf093..48c300d938a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HardcodedStackVersionPropertiesCheck.java @@ -46,7 +46,7 @@ @Singleton @UpgradeCheck( order = 98.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HardcodedStackVersionPropertiesCheck extends AbstractCheckDescriptor { public HardcodedStackVersionPropertiesCheck() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HealthCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HealthCheck.java index f08be16fe73..71dd61f5362 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HealthCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HealthCheck.java @@ -49,7 +49,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HealthCheck extends AbstractCheckDescriptor { private static final List ALERT_STATES = asList(WARNING, CRITICAL); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveDynamicServiceDiscoveryCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveDynamicServiceDiscoveryCheck.java index 34e5bfdd381..c631013cf52 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveDynamicServiceDiscoveryCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HiveDynamicServiceDiscoveryCheck.java @@ -44,7 +44,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, order = 20.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS }) public class HiveDynamicServiceDiscoveryCheck extends AbstractCheckDescriptor { static final String HIVE_DYNAMIC_SERVICE_DISCOVERY_ENABLED_KEY = "hive.dynamic-service.discovery.enabled.key"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostMaintenanceModeCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostMaintenanceModeCheck.java index 3125c262166..71342f14b1f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostMaintenanceModeCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostMaintenanceModeCheck.java @@ -43,7 +43,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.MAINTENANCE_MODE, order = 7.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HostMaintenanceModeCheck extends AbstractCheckDescriptor { public static final String KEY_CANNOT_START_HOST_ORDERED = "cannot_upgrade_mm_hosts"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java index 3218807ed9f..c957cbfc24e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsHeartbeatCheck.java @@ -45,7 +45,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.LIVELINESS, order = 1.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HostsHeartbeatCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsMasterMaintenanceCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsMasterMaintenanceCheck.java index 50948433ac2..8cae02ab8da 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsMasterMaintenanceCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsMasterMaintenanceCheck.java @@ -45,7 +45,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.MAINTENANCE_MODE, order = 5.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HostsMasterMaintenanceCheck extends AbstractCheckDescriptor { static final String KEY_NO_UPGRADE_NAME = "no_upgrade_name"; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java index ae36c658890..a03db614a64 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/HostsRepositoryVersionCheck.java @@ -45,7 +45,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.REPOSITORY_VERSION, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class HostsRepositoryVersionCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/InstallPackagesCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/InstallPackagesCheck.java index 3e02572f4b8..7dbff3c3222 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/InstallPackagesCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/InstallPackagesCheck.java @@ -45,7 +45,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, order = 3.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class InstallPackagesCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/PreviousUpgradeCompleted.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/PreviousUpgradeCompleted.java index 0292b721ec1..cab1ea4da10 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/PreviousUpgradeCompleted.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/PreviousUpgradeCompleted.java @@ -40,7 +40,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, order = 4.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class PreviousUpgradeCompleted extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/RangerPasswordCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/RangerPasswordCheck.java index 79f52d73754..c5eca750b7b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/RangerPasswordCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/RangerPasswordCheck.java @@ -53,7 +53,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.CONFIGURATION_WARNING, order = 23.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class RangerPasswordCheck extends AbstractCheckDescriptor { private static final Logger LOG = LoggerFactory.getLogger(RangerPasswordCheck.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/RequiredServicesInRepositoryCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/RequiredServicesInRepositoryCheck.java index ceed53c1b30..c0cf2f77844 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/RequiredServicesInRepositoryCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/RequiredServicesInRepositoryCheck.java @@ -45,7 +45,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.REPOSITORY_VERSION, order = 1.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }, + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }, orchestration = { RepositoryType.PATCH, RepositoryType.MAINT, RepositoryType.SERVICE }) public class RequiredServicesInRepositoryCheck extends AbstractCheckDescriptor { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java index 1126424f161..be0edda8b2c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServiceCheckValidityCheck.java @@ -57,7 +57,7 @@ @Singleton @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ServiceCheckValidityCheck extends AbstractCheckDescriptor { private static final Logger LOG = LoggerFactory.getLogger(ServiceCheckValidityCheck.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesMaintenanceModeCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesMaintenanceModeCheck.java index aa7b6db0334..8d7cd1b3137 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesMaintenanceModeCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesMaintenanceModeCheck.java @@ -37,7 +37,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.MAINTENANCE_MODE, order = 6.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ServicesMaintenanceModeCheck extends AbstractCheckDescriptor { /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesUpCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesUpCheck.java index 610632579dd..509da3dddcb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesUpCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesUpCheck.java @@ -67,7 +67,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.LIVELINESS, order = 2.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class ServicesUpCheck extends AbstractCheckDescriptor { private static final float SLAVE_THRESHOLD = 0.5f; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/checks/VersionMismatchCheck.java b/ambari-server/src/main/java/org/apache/ambari/server/checks/VersionMismatchCheck.java index 1dcad5fb05c..997869f1fa5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/checks/VersionMismatchCheck.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/checks/VersionMismatchCheck.java @@ -43,7 +43,7 @@ @UpgradeCheck( group = UpgradeCheckGroup.COMPONENT_VERSION, order = 7.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) public class VersionMismatchCheck extends AbstractCheckDescriptor { public VersionMismatchCheck() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/HostResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/HostResponse.java index 9d02a0bbcc5..2c40b059954 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/HostResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/HostResponse.java @@ -146,6 +146,29 @@ public class HostResponse { private MaintenanceState maintenanceState; + /** + * Summary information of all hosts in a cluster or multiple clusters + * + * The response json schema is: + * { + * "Hosts" : { + * "cluster_name" : "c1", + * "summary" : [ + * { + * "operating_systems" : [ + * { + * "centos6" : 2 + * }, + * { + * "centos7" : 5 + * } + * ] + * } + * ] + * } + */ + private List hostsSummary; + public HostResponse(String hostname, String clusterName, String ipv4, int cpuCount, int phCpuCount, String osArch, String osType, long totalMemBytes, @@ -421,6 +444,21 @@ public RecoveryReport getRecoveryReport() { return recoveryReport; } + /** + * Set the hostsSummary + */ + public void setHostsSummary(List hostsSummary) { + this.hostsSummary = hostsSummary; + } + + /** + * Get the aggregation info of hosts in a cluster or in multiple clusters + */ + @ApiModelProperty(name = HostResourceProvider.SUMMARY_PROPERTY_ID) + public List getHostsSummary() { + return hostsSummary; + } + /** * Set the detailed recovery report */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackRequest.java index a705d186a32..49d0caacc99 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackRequest.java @@ -26,25 +26,25 @@ */ public class MpackRequest { - private Long mpackId; + private Long id; + private Long registryId; private String mpackName; private String mpackVersion; private String mpackUri; - private Long registryId; - public MpackRequest(Long mpackId) { - this.setMpackId(mpackId); + public MpackRequest(Long id) { + this.setId(id); } public MpackRequest() { } - public Long getMpackId() { - return mpackId; + public Long getId() { + return id; } - public void setMpackId(Long mpackId) { - this.mpackId = mpackId; + public void setId(Long id) { + this.id = id; } public String getMpackName() { @@ -82,7 +82,7 @@ public void setRegistryId(Long registryId) { @Override public int hashCode() { int result = 1; - result = 31 + getMpackId().hashCode(); + result = 31 + getId().hashCode(); return result; } @@ -96,11 +96,11 @@ public boolean equals(Object obj) { } MpackRequest mpackRequest = (MpackRequest) obj; - if (mpackId == null) { - if (mpackRequest.mpackId != null) { + if (id == null) { + if (mpackRequest.id != null) { return false; } - } else if (!mpackId.equals(mpackRequest.mpackId)) { + } else if (!id.equals(mpackRequest.id)) { return false; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackResponse.java index ce70f8a8f15..0755c162076 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/MpackResponse.java @@ -27,86 +27,93 @@ */ public class MpackResponse { - private Long mpackId; - private String mpackVersion; + private Long id; + private String mpackId; private String mpackName; + private String mpackVersion; private String mpackUri; private Long registryId; private String stackId; private String description; public MpackResponse(Mpack mpack) { + this.id = mpack.getResourceId(); this.mpackId = mpack.getMpackId(); + this.mpackName = mpack.getName(); this.mpackVersion = mpack.getVersion(); this.mpackUri = mpack.getMpackUri(); - this.mpackName = mpack.getName(); this.registryId = mpack.getRegistryId(); - this.stackId = mpack.getStackId(); this.description = mpack.getDescription(); } - public String getMpackVersion() { - return mpackVersion; + public Long getId() { + return id; } - public String getMpackUri() { - return mpackUri; + public void setId(Long id) { + this.id = id; } - public Long getMpackId() { - return mpackId; + public Long getRegistryId() { + return registryId; } - public String getStackId() { - return stackId; + public void setRegistryId(Long registryId) { + this.registryId = registryId; } - public void setStackId(String stackId) { - this.stackId = stackId; + public String getMpackId() { + return mpackId; + } + + public void setMpackId(String mpackId) { + this.mpackId = mpackId; } public String getMpackName() { return mpackName; } - public Long getRegistryId() { - return registryId; + public void setMpackName(String mpackName) { + this.mpackName = mpackName; } - public void setMpackVersion(String mpackVersion) { - this.mpackVersion = mpackVersion; + public String getDescription() { + return description; } - public void setMpackName(String mpackName) { - this.mpackName = mpackName; + public void setDescription(String description) { + this.description = description; } - public void setMpackUri(String mpackUri) { - this.mpackUri = mpackUri; + public String getMpackVersion() { + return mpackVersion; } - public void setRegistryId(Long registryId) { - this.registryId = registryId; + public void setMpackVersion(String mpackVersion) { + this.mpackVersion = mpackVersion; } - public void setMpackId(Long mpackId) { - this.mpackId = mpackId; + public String getMpackUri() { + return mpackUri; } - - public String getDescription() { - return description; + public void setMpackUri(String mpackUri) { + this.mpackUri = mpackUri; } - public void setDescription(String description) { - this.description = description; + public String getStackId() { + return stackId; } + public void setStackId(String stackId) { + this.stackId = stackId; + } @Override public int hashCode() { int result = 1; - result = 31 + getMpackId().hashCode(); + result = 31 + getId().hashCode(); return result; } @@ -119,7 +126,7 @@ public boolean equals(Object obj) { return true; } MpackResponse MpackResponse = (MpackResponse) obj; - return getMpackId().equals(MpackResponse.getMpackId()); + return getId().equals(MpackResponse.getId()); } public interface MpackResponseWrapper extends ApiModel { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackResponse.java index 1bc96595de4..9943024b2b1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackResponse.java @@ -26,25 +26,25 @@ */ public class RegistryMpackResponse { private Long registryId; + private String mpackId; private String mpackName; - private String mpackDisplayName; private String mpackDescription; - private String mpackLogoUrl; + private String mpackLogoUri; /** * Constructor * @param registryId registry id + * @param mpackId mpack id * @param mpackName mpack name - * @param mpackDisplayName mpack Display Name * @param mpackDescription mpack description - * @param mpackLogoUrl mpack logo url + * @param mpackLogoUri mpack logo uri */ - public RegistryMpackResponse(Long registryId, String mpackName, String mpackDisplayName, String mpackDescription, String mpackLogoUrl) { + public RegistryMpackResponse(Long registryId, String mpackId, String mpackName, String mpackDescription, String mpackLogoUri) { this.registryId = registryId; this.mpackName = mpackName; - this.mpackDisplayName = mpackDisplayName; + this.mpackId = mpackId; this.mpackDescription = mpackDescription; - this.mpackLogoUrl = mpackLogoUrl; + this.mpackLogoUri = mpackLogoUri; } /** @@ -55,6 +55,15 @@ public Long getRegistryId() { return registryId; } + /** + * Get mpack id + * @return mpack id + */ + public String getMpackId() { + return mpackId; + } + + /** * Get mpack name * @return mpack name @@ -63,12 +72,6 @@ public String getMpackName() { return mpackName; } - /** - * Get mpack summary - * @return - */ - public String getMpackDisplayName() {return mpackDisplayName;} - /** * Get mpack description * @return mpack description @@ -78,11 +81,11 @@ public String getMpackDescription() { } /** - * Get mpack logo url - * @return mpack logo url + * Get mpack logo uri + * @return mpack logo uri */ - public String getMpackLogoUrl() { - return mpackLogoUrl; + public String getMpackLogoUri() { + return mpackLogoUri; } /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackVersionResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackVersionResponse.java index 5929c4c9987..829ed5e3eda 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackVersionResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryMpackVersionResponse.java @@ -20,8 +20,8 @@ import java.util.List; import org.apache.ambari.server.controller.internal.RegistryMpackVersionResourceProvider; -import org.apache.ambari.server.registry.RegistryMpackCompatiblity; -import org.apache.ambari.server.registry.RegistryMpackService; +import org.apache.ambari.server.registry.RegistryMpackDependency; +import org.apache.ambari.server.state.Module; import io.swagger.annotations.ApiModelProperty; @@ -30,82 +30,82 @@ */ public class RegistryMpackVersionResponse { private Long registryId; + private String mpackId; private String mpackName; + private String mpackDescription; private String mpackVersion; - private String mpackBuildNumber; - private String stackId; - private String mpackUrl; - private String mpackDocUrl; - private List mpackServices; - private List compatibleMpacks; + private String mpackUri; + private String mpackDocUri; + private String mpackLogoUri; + private List dependencies; + private List modules; /** * Constructor * @param registryId registry id + * @param mpackId mpack id * @param mpackName mpack name + * @param mpackDescription mpack description * @param mpackVersion mpack version - * @param mpackBuildNumber mpack build number - * @param mpackUrl mpack download url - * @param mpackDocUrl mpack documentation url - * @param mpackServices list of mpack services - * @param compatibleMpacks list of compatible mpacks - * @param stackId stack id of the mpack version + * @param mpackUri mpack uri + * @param mpackDocUri mpack documentation uri + * @param mpackLogoUri mpack logo uri + * @param dependencies list of mpack dependencies + * @param modules list of modules in the mpack */ - public RegistryMpackVersionResponse( - Long registryId, String mpackName, String mpackVersion, String mpackBuildNumber, - String mpackUrl, String mpackDocUrl, - List mpackServices, - List compatibleMpacks, String stackId) { + public RegistryMpackVersionResponse(Long registryId, String mpackId, String mpackName, String mpackDescription, + String mpackVersion, String mpackUri, String mpackDocUri, String mpackLogoUri, + List dependencies, List modules) { this.registryId = registryId; + this.mpackId = mpackId; this.mpackName = mpackName; + this.mpackDescription = mpackDescription; this.mpackVersion = mpackVersion; - this.mpackBuildNumber = mpackBuildNumber; - this.mpackUrl = mpackUrl; - this.mpackDocUrl = mpackDocUrl; - this.mpackServices = mpackServices; - this.compatibleMpacks = compatibleMpacks; - this.stackId = stackId; + this.mpackUri = mpackUri; + this.mpackDocUri = mpackDocUri; + this.mpackLogoUri = mpackLogoUri; + this.dependencies = dependencies; + this.modules = modules; } public Long getRegistryId() { return registryId; } - public String getMpackName() { - return mpackName; + public String getMpackId() { + return mpackId; } - public String getMpackVersion() { - return mpackVersion; + public String getMpackName() { + return mpackName; } - public String getMpackBuildNumber() { - return mpackBuildNumber; + public String getMpackDescription() { + return mpackDescription; } - public String getMpackUrl() { - return mpackUrl; + public String getMpackVersion() { + return mpackVersion; } - public String getMpackDocUrl() { - return mpackDocUrl; + public String getMpackUri() { + return mpackUri; } - public List getMpackServices() { - return mpackServices; + public String getMpackDocUri() { + return mpackDocUri; } - public List getCompatibleMpacks() { - return compatibleMpacks; + public String getMpackLogoUri() { + return mpackLogoUri; } - - public String getStackId() { - return stackId; + public List getDependencies() { + return dependencies; } - public void setStackId(String stackId) { - this.stackId = stackId; + public List getModules() { + return modules; } @Override diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryResponse.java index 729451fdeb4..b338fd57ad0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/RegistryResponse.java @@ -67,7 +67,6 @@ public String getRegistryUri() { return registryUri; } - public void setRegistryUri(String registryUri) { this.registryUri = registryUri; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupRequest.java index 38fba97b0f2..b9f94cecac0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupRequest.java @@ -23,10 +23,12 @@ public class ServiceGroupRequest { private String clusterName; // REF private String serviceGroupName; // GET/CREATE/UPDATE/DELETE + private String version; // Associated stack version info - public ServiceGroupRequest(String clusterName, String serviceGroupName) { + public ServiceGroupRequest(String clusterName, String serviceGroupName, String version) { this.clusterName = clusterName; this.serviceGroupName = serviceGroupName; + this.version = version; } /** @@ -57,9 +59,25 @@ public void setServiceGroupName(String serviceGroupName) { this.serviceGroupName = serviceGroupName; } + /** + * @return the servicegroup version + */ + public String getVersion() { + return version; + } + + /** + * @param version the servicegroup version to set + */ + public void setVersion(String version) { + this.version = version; + } + @Override public String toString() { - return String.format("clusterName=%s, serviceGroupName=%s", clusterName, serviceGroupName); + StringBuilder sb = new StringBuilder(); + sb.append("clusterName=").append(clusterName).append(", serviceGroupName=").append(serviceGroupName).append(", version=").append(version); + return sb.toString(); } @Override @@ -73,12 +91,11 @@ public boolean equals(Object obj) { ServiceGroupRequest other = (ServiceGroupRequest) obj; - return Objects.equals(clusterName, other.clusterName) && - Objects.equals(serviceGroupName, other.serviceGroupName); + return Objects.equals(clusterName, other.clusterName) && Objects.equals(serviceGroupName, other.serviceGroupName) && Objects.equals(version, other.version); } @Override public int hashCode() { - return Objects.hash(clusterName, serviceGroupName); + return Objects.hash(clusterName, serviceGroupName, version); } } \ No newline at end of file diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupResponse.java index 6b7c32fa547..afd7f248b61 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupResponse.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.controller; +import java.util.Objects; + import io.swagger.annotations.ApiModelProperty; public class ServiceGroupResponse { @@ -26,12 +28,15 @@ public class ServiceGroupResponse { private Long serviceGroupId; private String clusterName; private String serviceGroupName; + private String version; - public ServiceGroupResponse(Long clusterId, String clusterName, Long serviceGroupId, String serviceGroupName) { + public ServiceGroupResponse(Long clusterId, String clusterName, Long serviceGroupId, String serviceGroupName, String version) { this.clusterId = clusterId; this.serviceGroupId = serviceGroupId; this.clusterName = clusterName; this.serviceGroupName = serviceGroupName; + this.version = version; + } /** @@ -90,27 +95,28 @@ public void setServiceGroupName(String serviceGroupName) { this.serviceGroupName = serviceGroupName; } + /** + * @return the servicegroup version (stackName-stackVersion) + */ + public String getVersion() { + return version; + } + + /** + * @param version the servicegroup version (stackName-stackVersion) + */ + public void setVersion(String version) { + this.version = version; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - ServiceGroupResponse that = (ServiceGroupResponse) o; - - if (clusterId != null ? - !clusterId.equals(that.clusterId) : that.clusterId != null) { - return false; - } - if (clusterName != null ? - !clusterName.equals(that.clusterName) : that.clusterName != null) { - return false; - } - if (serviceGroupName != null ? - !serviceGroupName.equals(that.serviceGroupName) : that.serviceGroupName != null) { - return false; - } - - return true; + ServiceGroupResponse other = (ServiceGroupResponse) o; + + return Objects.equals(clusterId, other.clusterId) && Objects.equals(clusterName, other.clusterName) && Objects.equals(serviceGroupName, other.serviceGroupName) && Objects.equals(version, other.version); } /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractAuthorizedResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractAuthorizedResourceProvider.java index 8d92dd164d1..11cb2dd7e8d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractAuthorizedResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractAuthorizedResourceProvider.java @@ -219,7 +219,7 @@ public void setRequiredDeleteAuthorizations(Set requiredDelet */ @Override public RequestStatus createResources(Request request) - throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException { + throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, NoSuchParentResourceException, IllegalArgumentException { Authentication authentication = AuthorizationHelper.getAuthentication(); if (authentication == null || !authentication.isAuthenticated()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java index 1dbe9032f6d..b7f25013980 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/DefaultProviderModule.java @@ -132,6 +132,8 @@ protected ResourceProvider createResourceProvider(Resource.Type type) { return new ArtifactResourceProvider(managementController); case RemoteCluster: return new RemoteClusterResourceProvider(); + case Host: + return new HostResourceProvider(managementController); default: LOGGER.debug("Delegating creation of resource provider for: {} to the AbstractControllerResourceProvider", type.getInternalType()); return AbstractControllerResourceProvider.getResourceProvider(type, managementController); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java index 7ecbfdc13bc..79f4233ebce 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostResourceProvider.java @@ -116,6 +116,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { public static final String STATE_PROPERTY_ID = "host_state"; public static final String TOTAL_MEM_PROPERTY_ID = "total_mem"; public static final String ATTRIBUTES_PROPERTY_ID = "attributes"; + public static final String SUMMARY_PROPERTY_ID = "summary"; public static final String HOST_CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_NAME_PROPERTY_ID; public static final String HOST_CPU_COUNT_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CPU_COUNT_PROPERTY_ID; @@ -139,7 +140,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { public static final String HOST_RECOVERY_SUMMARY_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + RECOVERY_SUMMARY_PROPERTY_ID; public static final String HOST_STATE_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + STATE_PROPERTY_ID; public static final String HOST_TOTAL_MEM_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + TOTAL_MEM_PROPERTY_ID; - public static final String HOST_ATTRIBUTES_PROPERTY_ID = PropertyHelper.getPropertyId(RESPONSE_KEY,ATTRIBUTES_PROPERTY_ID); + public static final String HOST_ATTRIBUTES_PROPERTY_ID = PropertyHelper.getPropertyId(RESPONSE_KEY, ATTRIBUTES_PROPERTY_ID); public static final String BLUEPRINT_PROPERTY_ID = "blueprint"; public static final String HOST_GROUP_PROPERTY_ID = "host_group"; @@ -154,6 +155,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { public static Map keyPropertyIds = ImmutableMap.builder() .put(Resource.Type.Host, HOST_HOST_NAME_PROPERTY_ID) .put(Resource.Type.Cluster, HOST_CLUSTER_NAME_PROPERTY_ID) + .put(Resource.Type.HostComponent, HOST_OS_TYPE_PROPERTY_ID) .build(); /** @@ -195,7 +197,7 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { /** * Create a new resource provider for the given management controller. * - * @param managementController the management controller + * @param managementController the management controller */ @AssistedInject HostResourceProvider(@Assisted AmbariManagementController managementController) { @@ -214,9 +216,9 @@ public class HostResourceProvider extends AbstractControllerResourceProvider { @Override protected RequestStatus createResourcesAuthorized(final Request request) throws SystemException, - UnsupportedPropertyException, - ResourceAlreadyExistsException, - NoSuchParentResourceException { + UnsupportedPropertyException, + ResourceAlreadyExistsException, + NoSuchParentResourceException { RequestStatusResponse createResponse = null; if (isHostGroupRequest(request)) { @@ -238,6 +240,11 @@ public Void invoke() throws AmbariException, AuthorizationException { @Override protected Set getResourcesAuthorized(Request request, Predicate predicate) throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { + return getHostResource(request, predicate); + } + + private Set getHostResource(Request request, Predicate predicate) + throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { final Set requests = new HashSet<>(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java index 2ed6f3d1338..a867d11602d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java @@ -69,19 +69,20 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { public static final String RESPONSE_KEY = "MpackInfo"; public static final String ALL_PROPERTIES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "*"; - public static final String MPACK_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_id"; + public static final String MPACK_RESOURCE_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "id"; public static final String REGISTRY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "registry_id"; + public static final String MPACK_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_id"; public static final String MPACK_NAME = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_name"; public static final String MPACK_VERSION = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_version"; + public static final String MPACK_DESCRIPTION = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_description"; public static final String MPACK_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_uri"; public static final String MODULES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "modules"; - public static final String MPACK_DESC = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "description"; public static final String STACK_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "stack_name"; public static final String STACK_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "stack_version"; private static Set pkPropertyIds = new HashSet<>( - Arrays.asList(MPACK_ID, STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID)); + Arrays.asList(MPACK_RESOURCE_ID, STACK_NAME_PROPERTY_ID, STACK_VERSION_PROPERTY_ID)); /** * The property ids for an mpack resource. @@ -104,18 +105,19 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider { static { // properties - PROPERTY_IDS.add(MPACK_ID); + PROPERTY_IDS.add(MPACK_RESOURCE_ID); PROPERTY_IDS.add(REGISTRY_ID); + PROPERTY_IDS.add(MPACK_ID); PROPERTY_IDS.add(MPACK_NAME); PROPERTY_IDS.add(MPACK_VERSION); PROPERTY_IDS.add(MPACK_URI); - PROPERTY_IDS.add(MPACK_DESC); + PROPERTY_IDS.add(MPACK_DESCRIPTION); PROPERTY_IDS.add(MODULES); PROPERTY_IDS.add(STACK_NAME_PROPERTY_ID); PROPERTY_IDS.add(STACK_VERSION_PROPERTY_ID); // keys - KEY_PROPERTY_IDS.put(Resource.Type.Mpack, MPACK_ID); + KEY_PROPERTY_IDS.put(Resource.Type.Mpack, MPACK_RESOURCE_ID); KEY_PROPERTY_IDS.put(Resource.Type.Stack, STACK_NAME_PROPERTY_ID); KEY_PROPERTY_IDS.put(Resource.Type.StackVersion, STACK_VERSION_PROPERTY_ID); @@ -147,11 +149,12 @@ public RequestStatus createResourcesAuthorized(final Request request) if (response != null) { notifyCreate(Resource.Type.Mpack, request); Resource resource = new ResourceImpl(Resource.Type.Mpack); + resource.setProperty(MPACK_RESOURCE_ID, response.getId()); resource.setProperty(MPACK_ID, response.getMpackId()); resource.setProperty(MPACK_NAME, response.getMpackName()); resource.setProperty(MPACK_VERSION, response.getMpackVersion()); resource.setProperty(MPACK_URI, response.getMpackUri()); - resource.setProperty(MPACK_DESC, response.getDescription()); + resource.setProperty(MPACK_DESCRIPTION, response.getDescription()); resource.setProperty(REGISTRY_ID, response.getRegistryId()); associatedResources.add(resource); return getRequestStatus(null, associatedResources); @@ -203,21 +206,21 @@ public MpackRequest getRequest(Request request) throws AmbariException { Set> properties = request.getProperties(); for (Map propertyMap : properties) { //Mpack Download url is either given in the request body or is fetched using the registry id - if (!propertyMap.containsKey(MPACK_URI) && !propertyMap.containsKey(REGISTRY_ID)) + if (!propertyMap.containsKey(MPACK_URI) && !propertyMap.containsKey(REGISTRY_ID)) { return null; - //Fetch Mpack Download Url using the given registry id - else if (!propertyMap.containsKey(MPACK_URI)) { + } else if (!propertyMap.containsKey(MPACK_URI)) { + // Retrieve mpack download url using the given registry id mpackRequest.setRegistryId(Long.valueOf((String) propertyMap.get(REGISTRY_ID))); mpackRequest.setMpackName((String) propertyMap.get(MPACK_NAME)); mpackRequest.setMpackVersion((String) propertyMap.get(MPACK_VERSION)); mpackRequest.setMpackUri(getMpackUri(mpackRequest)); } - //Directly download the mpack using the given url - else + else { + //Directly download the mpack using the given url mpackRequest.setMpackUri((String) propertyMap.get(MPACK_URI)); + } } return mpackRequest; - } /*** @@ -231,7 +234,7 @@ private String getMpackUri(MpackRequest mpackRequest) throws AmbariException { Registry registry = getManagementController().getRegistry(mpackRequest.getRegistryId()); RegistryMpack registryMpack = registry.getRegistryMpack(mpackRequest.getMpackName()); RegistryMpackVersion registryMpackVersion = registryMpack.getMpackVersion(mpackRequest.getMpackVersion()); - return registryMpackVersion.getMpackUrl(); + return registryMpackVersion.getMpackUri(); } @Override @@ -249,11 +252,12 @@ public Set getResources(Request request, Predicate predicate) } for (MpackResponse response : responses){ Resource resource = new ResourceImpl(Resource.Type.Mpack); + resource.setProperty(MPACK_RESOURCE_ID, response.getId()); resource.setProperty(MPACK_ID, response.getMpackId()); resource.setProperty(MPACK_NAME, response.getMpackName()); resource.setProperty(MPACK_VERSION, response.getMpackVersion()); resource.setProperty(MPACK_URI, response.getMpackUri()); - resource.setProperty(MPACK_DESC, response.getDescription()); + resource.setProperty(MPACK_DESCRIPTION, response.getDescription()); resource.setProperty(REGISTRY_ID, response.getRegistryId()); results.add(resource); } @@ -264,16 +268,17 @@ public Set getResources(Request request, Predicate predicate) String stackName = (String) propertyMap.get(STACK_NAME_PROPERTY_ID); String stackVersion = (String) propertyMap.get(STACK_VERSION_PROPERTY_ID); StackEntity stackEntity = stackDAO.find(stackName, stackVersion); - mpackId = stackEntity.getCurrentMpackId(); + mpackId = stackEntity.getMpackId(); if (mpackId != null) { MpackResponse response = getManagementController().getMpack(mpackId); Resource resource = new ResourceImpl(Resource.Type.Mpack); if (null != response) { + resource.setProperty(MPACK_RESOURCE_ID, response.getId()); resource.setProperty(MPACK_ID, response.getMpackId()); resource.setProperty(MPACK_NAME, response.getMpackName()); resource.setProperty(MPACK_VERSION, response.getMpackVersion()); resource.setProperty(MPACK_URI, response.getMpackUri()); - resource.setProperty(MPACK_DESC, response.getDescription()); + resource.setProperty(MPACK_DESCRIPTION, response.getDescription()); resource.setProperty(REGISTRY_ID, response.getRegistryId()); resource.setProperty(STACK_NAME_PROPERTY_ID, stackName); resource.setProperty(STACK_VERSION_PROPERTY_ID, stackVersion); @@ -283,21 +288,22 @@ public Set getResources(Request request, Predicate predicate) return results; } - if (propertyMap.containsKey(MPACK_ID)) { - Object objMpackId = propertyMap.get(MPACK_ID); + if (propertyMap.containsKey(MPACK_RESOURCE_ID)) { + Object objMpackId = propertyMap.get(MPACK_RESOURCE_ID); if (objMpackId != null) mpackId = Long.valueOf((String) objMpackId); MpackResponse response = getManagementController().getMpack(mpackId); Resource resource = new ResourceImpl(Resource.Type.Mpack); if (null != response) { + resource.setProperty(MPACK_RESOURCE_ID, response.getId()); resource.setProperty(MPACK_ID, response.getMpackId()); resource.setProperty(MPACK_NAME, response.getMpackName()); resource.setProperty(MPACK_VERSION, response.getMpackVersion()); resource.setProperty(MPACK_URI, response.getMpackUri()); - resource.setProperty(MPACK_DESC, response.getDescription()); + resource.setProperty(MPACK_DESCRIPTION, response.getDescription()); resource.setProperty(REGISTRY_ID, response.getRegistryId()); - List modules = getManagementController().getModules(response.getMpackId()); + List modules = getManagementController().getModules(response.getId()); resource.setProperty(MODULES, modules); results.add(resource); } @@ -325,8 +331,8 @@ protected RequestStatus deleteResourcesAuthorized(final Request request, Predica if (getManagementController().getClusters().getClusters().size() > 0) { throw new SystemException("Delete request cannot be completed since there is a cluster deployed"); } else { - if (propertyMap.containsKey(MPACK_ID)) { - Object objMpackId = propertyMap.get(MPACK_ID); + if (propertyMap.containsKey(MPACK_RESOURCE_ID)) { + Object objMpackId = propertyMap.get(MPACK_RESOURCE_ID); if (objMpackId != null) { mpackId = Long.valueOf((String) objMpackId); LOG.info("Deleting Mpack, id = " + mpackId.toString()); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackResourceProvider.java index 2e1c23cb290..fa0f62b6633 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackResourceProvider.java @@ -49,13 +49,13 @@ public class RegistryMpackResourceProvider extends AbstractControllerResourcePro public static final String RESPONSE_KEY = "RegistryMpackInfo"; public static final String ALL_PROPERTIES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "*"; public static final String REGISTRY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "registry_id"; + public static final String REGISTRY_MPACK_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_id"; public static final String REGISTRY_MPACK_NAME = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_name"; - public static final String REGISTRY_MPACK_DISPLAY_NAME = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_display_name"; public static final String REGISTRY_MPACK_DESC = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_description"; - public static final String REGISTRY_MPACK_LOGO_URL = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_logo_url"; + public static final String REGISTRY_MPACK_LOGO_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_logo_uri"; private static Set pkPropertyIds = new HashSet<>( - Arrays.asList(REGISTRY_ID, REGISTRY_MPACK_NAME)); + Arrays.asList(REGISTRY_ID, REGISTRY_MPACK_ID, REGISTRY_MPACK_NAME)); /** * The property ids for a software registry resource. @@ -70,10 +70,10 @@ public class RegistryMpackResourceProvider extends AbstractControllerResourcePro static { // properties PROPERTY_IDS.add(REGISTRY_ID); + PROPERTY_IDS.add(REGISTRY_MPACK_ID); PROPERTY_IDS.add(REGISTRY_MPACK_NAME); - PROPERTY_IDS.add(REGISTRY_MPACK_DISPLAY_NAME); PROPERTY_IDS.add(REGISTRY_MPACK_DESC); - PROPERTY_IDS.add(REGISTRY_MPACK_LOGO_URL); + PROPERTY_IDS.add(REGISTRY_MPACK_LOGO_URI); // keys KEY_PROPERTY_IDS.put(Resource.Type.Registry, REGISTRY_ID); @@ -130,10 +130,10 @@ public Set invoke() throws AmbariException { for (RegistryMpackResponse response : responses) { Resource resource = new ResourceImpl(Resource.Type.RegistryMpack); setResourceProperty(resource, REGISTRY_ID, response.getRegistryId(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_ID, response.getMpackId(), requestedIds); setResourceProperty(resource, REGISTRY_MPACK_NAME, response.getMpackName(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_DISPLAY_NAME, response.getMpackDisplayName(), requestedIds); setResourceProperty(resource, REGISTRY_MPACK_DESC, response.getMpackDescription(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_LOGO_URL, response.getMpackLogoUrl(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_LOGO_URI, response.getMpackLogoUri(), requestedIds); resources.add(resource); } return resources; @@ -184,10 +184,10 @@ private Set getRegistryMpacks(RegistryMpackRequest reques for (RegistryMpack registryMpack : registry.getRegistryMpacks()) { RegistryMpackResponse response = new RegistryMpackResponse( registry.getRegistryId(), + registryMpack.getMpackId(), registryMpack.getMpackName(), - registryMpack.getMpackDisplayName(), registryMpack.getMpackDescription(), - registryMpack.getMpackLogoUrl()); + registryMpack.getMpackLogoUri()); responses.add(response); } } else { @@ -195,10 +195,10 @@ private Set getRegistryMpacks(RegistryMpackRequest reques if(registryMpack != null) { RegistryMpackResponse response = new RegistryMpackResponse( registry.getRegistryId(), + registryMpack.getMpackId(), registryMpack.getMpackName(), - registryMpack.getMpackDisplayName(), registryMpack.getMpackDescription(), - registryMpack.getMpackLogoUrl()); + registryMpack.getMpackLogoUri()); responses.add(response); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackVersionResourceProvider.java index 4b5393d7652..7dfddb0a8dc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/RegistryMpackVersionResourceProvider.java @@ -58,20 +58,18 @@ public class RegistryMpackVersionResourceProvider extends AbstractControllerReso public static final String ALL_PROPERTIES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "*"; public static final String REGISTRY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "registry_id"; + public static final String REGISTRY_MPACK_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_id"; public static final String REGISTRY_MPACK_NAME = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_name"; + public static final String REGISTRY_MPACK_DESCRIPTION = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_description"; public static final String REGISTRY_MPACK_VERSION = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_version"; - public static final String REGISTRY_MPACK_BUILDNUM = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_buildnum"; - public static final String REGISTRY_MPACK_URL = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_url"; - public static final String REGISTRY_MPACK_DOC_URL = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_doc_url"; - public static final String REGISTRY_MPACK_SERVICES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "services"; - public static final String REGISTRY_MPACK_COMPATIBLE_MPACKS = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "compatible_mpacks"; - public static final String REGISTRY_MPACK_STACK_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "stack_name"; - public static final String REGISTRY_MPACK_STACK_VERSION_PROPERTY_ID = - RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "stack_version"; - + public static final String REGISTRY_MPACK_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_uri"; + public static final String REGISTRY_MPACK_DOC_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_doc_uri"; + public static final String REGISTRY_MPACK_LOGO_URI = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_logo_uri"; + public static final String REGISTRY_MPACK_DEPENDENCIES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "mpack_dependencies"; + public static final String REGISTRY_MPACK_MODULES = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "modules"; private static Set pkPropertyIds = new HashSet<>( - Arrays.asList(REGISTRY_ID, REGISTRY_MPACK_NAME)); + Arrays.asList(REGISTRY_ID, REGISTRY_MPACK_ID, REGISTRY_MPACK_NAME)); /** * The property ids for a software registry resource. @@ -91,17 +89,15 @@ public class RegistryMpackVersionResourceProvider extends AbstractControllerReso static { // properties PROPERTY_IDS.add(REGISTRY_ID); + PROPERTY_IDS.add(REGISTRY_MPACK_ID); PROPERTY_IDS.add(REGISTRY_MPACK_NAME); + PROPERTY_IDS.add(REGISTRY_MPACK_DESCRIPTION); PROPERTY_IDS.add(REGISTRY_MPACK_VERSION); - PROPERTY_IDS.add(REGISTRY_MPACK_BUILDNUM); - PROPERTY_IDS.add(REGISTRY_MPACK_URL); - PROPERTY_IDS.add(REGISTRY_MPACK_DOC_URL); - PROPERTY_IDS.add(REGISTRY_MPACK_SERVICES); - PROPERTY_IDS.add(REGISTRY_MPACK_COMPATIBLE_MPACKS); - PROPERTY_IDS.add(REGISTRY_MPACK_STACK_NAME_PROPERTY_ID); - PROPERTY_IDS.add(REGISTRY_MPACK_STACK_VERSION_PROPERTY_ID); - - + PROPERTY_IDS.add(REGISTRY_MPACK_URI); + PROPERTY_IDS.add(REGISTRY_MPACK_DOC_URI); + PROPERTY_IDS.add(REGISTRY_MPACK_LOGO_URI); + PROPERTY_IDS.add(REGISTRY_MPACK_DEPENDENCIES); + PROPERTY_IDS.add(REGISTRY_MPACK_MODULES); // keys KEY_PROPERTY_IDS.put(Resource.Type.Registry, REGISTRY_ID); KEY_PROPERTY_IDS.put(Resource.Type.RegistryMpack, REGISTRY_MPACK_NAME); @@ -157,15 +153,15 @@ public Set invoke() throws AmbariException { for (RegistryMpackVersionResponse response : responses) { Resource resource = new ResourceImpl(Resource.Type.RegistryMpackVersion); setResourceProperty(resource, REGISTRY_ID, response.getRegistryId(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_ID, response.getMpackId(), requestedIds); setResourceProperty(resource, REGISTRY_MPACK_NAME, response.getMpackName(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_DESCRIPTION, response.getMpackDescription(), requestedIds); setResourceProperty(resource, REGISTRY_MPACK_VERSION, response.getMpackVersion(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_BUILDNUM, response.getMpackBuildNumber(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_URL, response.getMpackUrl(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_DOC_URL, response.getMpackDocUrl(), requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_SERVICES, response.getMpackServices(), requestedIds); - String[] stackid = ((String) response.getStackId()).split("-"); - setResourceProperty(resource, REGISTRY_MPACK_STACK_NAME_PROPERTY_ID, stackid[0], requestedIds); - setResourceProperty(resource, REGISTRY_MPACK_STACK_VERSION_PROPERTY_ID, stackid[1], requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_URI, response.getMpackUri(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_DOC_URI, response.getMpackDocUri(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_LOGO_URI, response.getMpackLogoUri(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_DEPENDENCIES, response.getDependencies(), requestedIds); + setResourceProperty(resource, REGISTRY_MPACK_MODULES, response.getModules(), requestedIds); sortedResources.add(resource); } sortedResources.sort(new Comparator() { @@ -244,14 +240,15 @@ private Set getRegistryMpackVersions(RegistryMpack for (RegistryMpackVersion registryMpackVersion : registryMpack.getMpackVersions()) { RegistryMpackVersionResponse response = new RegistryMpackVersionResponse( registry.getRegistryId(), + registryMpack.getMpackId(), registryMpack.getMpackName(), + registryMpack.getMpackDescription(), registryMpackVersion.getMpackVersion(), - registryMpackVersion.getMpackBuildNumber(), - registryMpackVersion.getMpackUrl(), - registryMpackVersion.getMpackDocUrl(), - registryMpackVersion.getMpackServices(), - registryMpackVersion.getCompatibleMpacks(), - registryMpackVersion.getMpackStackId()); + registryMpackVersion.getMpackUri(), + registryMpackVersion.getMpackDocUri(), + registryMpack.getMpackLogoUri(), + registryMpackVersion.getDependencies(), + registryMpackVersion.getModules()); responses.add(response); } } else { @@ -259,14 +256,15 @@ private Set getRegistryMpackVersions(RegistryMpack if(registryMpackVersion != null) { RegistryMpackVersionResponse response = new RegistryMpackVersionResponse( registry.getRegistryId(), + registryMpack.getMpackId(), registryMpack.getMpackName(), + registryMpack.getMpackDescription(), registryMpackVersion.getMpackVersion(), - registryMpackVersion.getMpackBuildNumber(), - registryMpackVersion.getMpackUrl(), - registryMpackVersion.getMpackDocUrl(), - registryMpackVersion.getMpackServices(), - registryMpackVersion.getCompatibleMpacks(), - registryMpackVersion.getMpackStackId()); + registryMpackVersion.getMpackUri(), + registryMpackVersion.getMpackDocUri(), + registryMpack.getMpackLogoUri(), + registryMpackVersion.getDependencies(), + registryMpackVersion.getModules()); responses.add(response); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProvider.java index e8bece58ca4..ee87b1b31f0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProvider.java @@ -54,6 +54,7 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ServiceGroup; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; @@ -78,6 +79,7 @@ public class ServiceGroupResourceProvider extends AbstractControllerResourceProv public static final String SERVICE_GROUP_CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "cluster_name"; public static final String SERVICE_GROUP_SERVICE_GROUP_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "service_group_id"; public static final String SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "service_group_name"; + public static final String SERVICE_GROUP_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + "version"; private static Set pkPropertyIds = @@ -103,10 +105,12 @@ public class ServiceGroupResourceProvider extends AbstractControllerResourceProv PROPERTY_IDS.add(SERVICE_GROUP_CLUSTER_NAME_PROPERTY_ID); PROPERTY_IDS.add(SERVICE_GROUP_SERVICE_GROUP_ID_PROPERTY_ID); PROPERTY_IDS.add(SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID); + PROPERTY_IDS.add(SERVICE_GROUP_VERSION_PROPERTY_ID); // keys KEY_PROPERTY_IDS.put(Resource.Type.Cluster, SERVICE_GROUP_CLUSTER_NAME_PROPERTY_ID); KEY_PROPERTY_IDS.put(Resource.Type.ServiceGroup, SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID); + KEY_PROPERTY_IDS.put(Resource.Type.Stack, SERVICE_GROUP_VERSION_PROPERTY_ID); } private Clusters clusters; @@ -136,7 +140,7 @@ protected RequestStatus createResourcesAuthorized(Request request) throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException, - NoSuchParentResourceException { + NoSuchParentResourceException, IllegalArgumentException{ final Set requests = new HashSet<>(); for (Map propertyMap : request.getProperties()) { @@ -145,7 +149,7 @@ protected RequestStatus createResourcesAuthorized(Request request) Set createServiceGroups = null; createServiceGroups = createResources(new Command>() { @Override - public Set invoke() throws AmbariException, AuthorizationException { + public Set invoke() throws AmbariException, AuthorizationException, IllegalArgumentException { return createServiceGroups(requests); } }); @@ -160,6 +164,7 @@ public Set invoke() throws AmbariException, AuthorizationE resource.setProperty(SERVICE_GROUP_CLUSTER_NAME_PROPERTY_ID, response.getClusterName()); resource.setProperty(SERVICE_GROUP_SERVICE_GROUP_ID_PROPERTY_ID, response.getServiceGroupId()); resource.setProperty(SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID, response.getServiceGroupName()); + resource.setProperty(SERVICE_GROUP_VERSION_PROPERTY_ID, response.getVersion()); associatedResources.add(resource); } @@ -199,6 +204,8 @@ public Set invoke() throws AmbariException { response.getServiceGroupId(), requestedIds); setResourceProperty(resource, SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID, response.getServiceGroupName(), requestedIds); + setResourceProperty(resource, SERVICE_GROUP_VERSION_PROPERTY_ID, + response.getVersion(), requestedIds); resources.add(resource); } return resources; @@ -268,13 +275,14 @@ protected Set getPKPropertyIds() { private ServiceGroupRequest getRequest(Map properties) { String clusterName = (String) properties.get(SERVICE_GROUP_CLUSTER_NAME_PROPERTY_ID); String serviceGroupName = (String) properties.get(SERVICE_GROUP_SERVICE_GROUP_NAME_PROPERTY_ID); - ServiceGroupRequest svcRequest = new ServiceGroupRequest(clusterName, serviceGroupName); + String version = (String) properties.get(SERVICE_GROUP_VERSION_PROPERTY_ID); + ServiceGroupRequest svcRequest = new ServiceGroupRequest(clusterName, serviceGroupName, version); return svcRequest; } // Create services from the given request. public synchronized Set createServiceGroups(Set requests) - throws AmbariException, AuthorizationException { + throws AmbariException, AuthorizationException, IllegalArgumentException { if (requests.isEmpty()) { LOG.warn("Received an empty requests set"); @@ -291,7 +299,7 @@ public synchronized Set createServiceGroups(Set requests, Clusters clusters) - throws AuthorizationException, AmbariException { + throws AuthorizationException, AmbariException, IllegalArgumentException { AmbariMetaInfo ambariMetaInfo = getManagementController().getAmbariMetaInfo(); Map> serviceGroupNames = new HashMap<>(); @@ -397,13 +405,25 @@ private void validateCreateRequests(Set requests, Clusters for (ServiceGroupRequest request : requests) { final String clusterName = request.getClusterName(); final String serviceGroupName = request.getServiceGroupName(); + String version = request.getVersion(); + //TODO: This should not happen, after UI changes the code, this check should be removed + if (StringUtils.isBlank(version)) { + try { + Cluster cluster = clusters.getCluster(clusterName); + StackId stackId = cluster.getCurrentStackVersion(); + request.setVersion(stackId.getStackId()); + } catch (ClusterNotFoundException e) { + throw new ParentObjectNotFoundException("Cluster " + clusterName + " does not exist: ", e); + } + } Validate.notNull(clusterName, "Cluster name should be provided when creating a service group"); Validate.notEmpty(serviceGroupName, "Service group name should be provided when creating a service group"); + Validate.notEmpty(request.getVersion(), "Stack version should be provided when creating a service group"); if (LOG.isDebugEnabled()) { LOG.debug("Received a createServiceGroup request" + - ", clusterName=" + clusterName + ", serviceGroupName=" + serviceGroupName + ", request=" + request); + ", clusterName=" + clusterName + ", serviceGroupName=" + serviceGroupName + ", version=" + version + ", request=" + request); } if (!AuthorizationHelper.isAuthorized(ResourceType.CLUSTER, @@ -422,12 +442,14 @@ private void validateCreateRequests(Set requests, Clusters } serviceGroupNames.get(clusterName).add(serviceGroupName); + // TODO: Validate stack version Cluster cluster; try { cluster = clusters.getCluster(clusterName); } catch (ClusterNotFoundException e) { throw new ParentObjectNotFoundException("Attempted to add a service group to a cluster which doesn't exist", e); } + try { ServiceGroup sg = cluster.getServiceGroup(serviceGroupName); if (sg != null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 8f4ae7c288e..f50ef747d00 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -720,7 +720,7 @@ protected UpgradeEntity createUpgrade(UpgradeContext upgradeContext) // Non Rolling Upgrades require a group with name "UPDATE_DESIRED_REPOSITORY_ID". // This is needed as a marker to indicate which version to use when an upgrade is paused. - if (pack.getType() == UpgradeType.NON_ROLLING) { + if (pack.getType() == UpgradeType.EXPRESS) { boolean foundUpdateDesiredRepositoryIdGrouping = false; for (UpgradeGroupHolder group : groups) { if (group.groupClass == UpdateStackGrouping.class) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java index 6795aef2923..f2749827b21 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackGenerator.java @@ -243,7 +243,6 @@ public void generateMpack() throws Exception { Mpack mpack = new Mpack(); mpack.setName(dstStackName); mpack.setVersion(vdf.release.version); - mpack.setStackId(dstStackId.getStackId()); mpack.setDescription(dstStackName + " Ambari Management Pack"); Map prereqs = new HashMap<>(); prereqs.put("min-ambari-version", "3.0.0.0"); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java index 300c22e7031..d77c5e1fa70 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java @@ -29,6 +29,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -46,7 +47,7 @@ import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.state.Module; import org.apache.ambari.server.state.Mpack; -import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.OsSpecific; import org.apache.ambari.server.state.stack.StackMetainfoXml; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -120,9 +121,10 @@ private void parseMpackDirectories() { "UTF-8"); Gson gson = new Gson(); Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class); - existingMpack.setMpackId(mpackEntity.getMpackId()); + existingMpack.setResourceId(mpackEntity.getId()); existingMpack.setMpackUri(mpackEntity.getMpackUri()); - mpackMap.put(mpackEntity.getMpackId(), existingMpack); + existingMpack.setRegistryId(mpackEntity.getRegistryId()); + mpackMap.put(mpackEntity.getId(), existingMpack); } } } @@ -157,7 +159,7 @@ public void setMpackMap(Map mpackMap) { public MpackResponse registerMpack(MpackRequest mpackRequest) throws IOException, IllegalArgumentException, ResourceAlreadyExistsException { - Long mpackId; + Long mpackResourceId; String mpackName = ""; String mpackVersion = ""; Mpack mpack = new Mpack(); @@ -169,27 +171,25 @@ public MpackResponse registerMpack(MpackRequest mpackRequest) if (mpackRequest.getRegistryId() != null) { mpackName = mpackRequest.getMpackName(); mpackVersion = mpackRequest.getMpackVersion(); - mpack.setRegistryId(mpackRequest.getRegistryId()); LOG.info("Mpack Registration via Registry :" + mpackName); mpack = downloadMpackMetadata(mpackRequest.getMpackUri()); + mpack.setRegistryId(mpackRequest.getRegistryId()); isValidMetadata = validateMpackInfo(mpackName, mpackVersion, mpack.getName(), mpack.getVersion()); if (isValidMetadata) { mpackTarPath = downloadMpack(mpackRequest.getMpackUri(), mpack.getDefinition()); createMpackDirectory(mpack); mpackDirectory = mpacksStaging + File.separator + mpack.getName() + File.separator + mpack.getVersion(); + } else { + String message = + "Incorrect information : Mismatch in - (" + mpackName + "," + mpack.getName() + ") or (" + mpackVersion + + "," + mpack.getVersion() + ")"; + throw new IllegalArgumentException(message); //Mismatch in information } - else { - String message = - "Incorrect information : Mismatch in - (" + mpackName + "," + mpack.getName() + ") or (" + mpackVersion - + "," + mpack.getVersion() + ")"; - throw new IllegalArgumentException(message); //Mismatch in information - } - } - //Mpack registration using direct download - else { + } else { + // Mpack registration using direct download mpack = downloadMpackMetadata(mpackRequest.getMpackUri()); mpackTarPath = downloadMpack(mpackRequest.getMpackUri(), mpack.getDefinition()); @@ -201,18 +201,18 @@ public MpackResponse registerMpack(MpackRequest mpackRequest) } extractMpackTar(mpack, mpackTarPath, mpackDirectory); mpack.setMpackUri(mpackRequest.getMpackUri()); - mpackId = populateDB(mpack); + mpackResourceId = populateDB(mpack); - if (mpackId != null) { - mpackMap.put(mpackId, mpack); - mpack.setMpackId(mpackId); + if (mpackResourceId != null) { + mpackMap.put(mpackResourceId, mpack); + mpack.setResourceId(mpackResourceId); populateStackDB(mpack); return new MpackResponse(mpack); - } else { - String message = "Mpack :" + mpackRequest.getMpackName() + " version: " + mpackRequest.getMpackVersion() - + " already exists in server"; - throw new ResourceAlreadyExistsException(message); } + String message = "Mpack :" + mpackRequest.getMpackName() + " version: " + mpackRequest.getMpackVersion() + + " already exists in server"; + throw new ResourceAlreadyExistsException(message); + } /*** @@ -328,6 +328,17 @@ private void generateMetainfo(File metainfoFile, Mpack mpack) throws IOException version.setActive(true); generatedMetainfo.setVersion(version); + //Add osSpecifics to the metainfo.xml + OsSpecific osSpecific = new OsSpecific("any"); + OsSpecific.Package pkg = new OsSpecific.Package(); + pkg.setName(mpack.getName().toLowerCase()); + ArrayList packageArrayList = new ArrayList<>(); + packageArrayList.add(pkg); + ArrayList osSpecificArrayList = new ArrayList<>(); + osSpecificArrayList.add(osSpecific); + osSpecific.addPackages(packageArrayList); + generatedMetainfo.setOsSpecifics(osSpecificArrayList); + Map prerequisites = mpack.getPrerequisites(); if (prerequisites != null && prerequisites.containsKey(MIN_JDK_PROPERTY)) { generatedMetainfo.setMinJdk(mpack.getPrerequisites().get(MIN_JDK_PROPERTY)); @@ -420,17 +431,8 @@ private Boolean createMpackDirectory(Mpack mpack) */ private void createSymLinks(Mpack mpack) throws IOException { - String stackId = mpack.getStackId(); - String stackName = ""; - String stackVersion = ""; - if (stackId == null) { - stackName = mpack.getName(); - stackVersion = mpack.getVersion(); - } else { - StackId id = new StackId(stackId); - stackName = id.getStackName(); - stackVersion = id.getStackVersion(); - } + String stackName = mpack.getName(); + String stackVersion = mpack.getVersion(); File stack = new File(stackRoot + "/" + stackName); Path stackPath = Paths.get(stackRoot + "/" + stackName + "/" + stackVersion); Path mpackPath = Paths.get(mpacksStaging + "/" + mpack.getName() + "/" + mpack.getVersion()); @@ -481,9 +483,8 @@ protected boolean validateMpackInfo( String actualMpackName, String actualMpackVersion) { - String strippedActualMpackVersion = actualMpackVersion.substring(0, actualMpackVersion.lastIndexOf('.')); if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion - .equalsIgnoreCase(strippedActualMpackVersion)) { + .equalsIgnoreCase(actualMpackVersion)) { return true; } else { LOG.info("Incorrect information : Mismatch in - (" + expectedMpackName + "," + actualMpackName + ") or (" @@ -504,8 +505,8 @@ protected Long populateDB(Mpack mpack) throws IOException { String mpackName = mpack.getName(); String mpackVersion = mpack.getVersion(); List resultSet = mpackDAO.findByNameVersion(mpackName, mpackVersion); - - if (resultSet.size() == 0) { + StackEntity stackEntity = stackDAO.find(mpackName, mpackVersion); + if (resultSet.size() == 0 && stackEntity == null) { LOG.info("Adding mpack {}-{} to the database", mpackName, mpackVersion); MpackEntity mpackEntity = new MpackEntity(); @@ -527,20 +528,9 @@ protected Long populateDB(Mpack mpack) throws IOException { * @param mpack * @throws IOException */ - protected void populateStackDB(Mpack mpack) throws IOException { - - String stackId = mpack.getStackId(); - String stackName = ""; - String stackVersion = ""; - if (stackId == null) { - stackName = mpack.getName(); - stackVersion = mpack.getVersion(); - } else { - StackId id = new StackId(stackId); - stackName = id.getStackName(); - stackVersion = id.getStackVersion(); - } - + protected void populateStackDB(Mpack mpack) throws IOException, ResourceAlreadyExistsException { + String stackName = mpack.getName(); + String stackVersion = mpack.getVersion(); StackEntity stackEntity = stackDAO.find(stackName, stackVersion); if (stackEntity == null) { LOG.info("Adding stack {}-{} to the database", stackName, stackVersion); @@ -548,13 +538,12 @@ protected void populateStackDB(Mpack mpack) throws IOException { stackEntity.setStackName(stackName); stackEntity.setStackVersion(stackVersion); - stackEntity.setCurrentMpackId(mpack.getMpackId()); + stackEntity.setMpackId(mpack.getResourceId()); stackDAO.create(stackEntity); } else { - LOG.info("Updating stack {}-{} to the database", stackName, stackVersion); - - stackEntity.setCurrentMpackId(mpack.getMpackId()); - stackDAO.merge(stackEntity); + String message = "Stack " + stackName + "-" + stackVersion + " already exists"; + LOG.error(message); + throw new ResourceAlreadyExistsException(message); } } @@ -586,7 +575,7 @@ public boolean removeMpack(MpackEntity mpackEntity, StackEntity stackEntity) thr LOG.info("Removing mpack :" + mpackName); - mpackMap.remove(mpackEntity.getMpackId()); + mpackMap.remove(mpackEntity.getId()); FileUtils.deleteDirectory(mpackDirToDelete); if (mpackDirectory.isDirectory()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MpackDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MpackDAO.java index 3228130c74f..43e2a7f89dd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MpackDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/MpackDAO.java @@ -56,19 +56,19 @@ public class MpackDAO { @Transactional public Long create(MpackEntity mpackEntity) { m_entityManagerProvider.get().persist(mpackEntity); - return mpackEntity.getMpackId(); + return mpackEntity.getId(); } /** * Gets an mpack with the specified ID. * - * @param mpackId - * the ID of the alert to retrieve. + * @param id + * the ID of the mpack to retrieve. * @return the mpack or {@code null} if none exists. */ @RequiresSession - public MpackEntity findById(long mpackId) { - return m_entityManagerProvider.get().find(MpackEntity.class, mpackId); + public MpackEntity findById(long id) { + return m_entityManagerProvider.get().find(MpackEntity.class, id); } /** @@ -99,8 +99,8 @@ public List findAll() { } @Transactional - public void removeById(Long mpackId) { - m_entityManagerProvider.get().remove(findById(mpackId)); + public void removeById(Long id) { + m_entityManagerProvider.get().remove(findById(id)); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StackDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StackDAO.java index 57327c60c63..559f47cd235 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StackDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/StackDAO.java @@ -196,7 +196,7 @@ public void removeByMpack(Long mpackId) { public StackEntity findByMpack(Long mpackId) { TypedQuery query = entityManagerProvider.get().createNamedQuery( "StackEntity.findByMpack", StackEntity.class); - query.setParameter("currentMpackId", mpackId); + query.setParameter("mpackId", mpackId); return daoUtils.selectOne(query); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/MpackEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/MpackEntity.java index d3f6f285c6e..879d24a1238 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/MpackEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/MpackEntity.java @@ -41,7 +41,7 @@ @Entity @TableGenerator(name = "mpack_id_generator", table = "ambari_sequences", pkColumnName = "sequence_name", valueColumnName = "sequence_value", pkColumnValue = "mpack_id_seq", initialValue = 1) @NamedQueries({ - @NamedQuery(name = "MpackEntity.findById", query = "SELECT mpack FROM MpackEntity mpack where mpack.mpackId = :mpackId"), + @NamedQuery(name = "MpackEntity.findById", query = "SELECT mpack FROM MpackEntity mpack where mpack.id = :id"), @NamedQuery(name = "MpackEntity.findAll", query = "SELECT mpack FROM MpackEntity mpack"), @NamedQuery(name = "MpackEntity.findByNameVersion", query = "SELECT mpack FROM MpackEntity mpack where mpack.mpackName = :mpackName and mpack.mpackVersion = :mpackVersion")}) @@ -50,7 +50,7 @@ public class MpackEntity { @Id @GeneratedValue(strategy = GenerationType.TABLE, generator = "mpack_id_generator") @Column(name = "id", nullable = false, updatable = false) - private Long mpackId; + private Long id; @Column(name = "registry_id", nullable = true, insertable = true, updatable = false, length = 10) private Long registryId; @@ -64,8 +64,8 @@ public class MpackEntity { @Column(name = "mpack_uri", nullable = false) private String mpackUri; - public Long getMpackId() { - return mpackId; + public Long getId() { + return id; } public Long getRegistryId() { @@ -84,8 +84,8 @@ public String getMpackUri() { return mpackUri; } - public void setMpackId(Long mpackId) { - this.mpackId = mpackId; + public void setId(Long id) { + this.id = id; } public void setRegistryId(Long registryId) { @@ -121,7 +121,7 @@ public boolean equals(Object object) { MpackEntity that = (MpackEntity) object; EqualsBuilder equalsBuilder = new EqualsBuilder(); - equalsBuilder.append(mpackId, that.mpackId); + equalsBuilder.append(id, that.id); equalsBuilder.append(mpackName, that.mpackName); equalsBuilder.append(mpackVersion, that.mpackVersion); return equalsBuilder.isEquals(); @@ -130,7 +130,7 @@ public boolean equals(Object object) { /** * Generates a hash for the mpack based on the following criteria: *
    - *
  • {@link #mpackId} + *
  • {@link #id} *
*

*

@@ -138,7 +138,7 @@ public boolean equals(Object object) { */ @Override public int hashCode() { - return Objects.hash(mpackId, mpackName, mpackVersion); + return Objects.hash(id, mpackName, mpackVersion); } /** @@ -147,7 +147,7 @@ public int hashCode() { @Override public String toString() { StringBuilder buffer = new StringBuilder("MpackEntity{"); - buffer.append("mpackId=").append(mpackId); + buffer.append("id=").append(id); if (null != registryId) { buffer.append(", registryId=").append(registryId); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceGroupEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceGroupEntity.java index 7d03e443ea6..48b7664b802 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceGroupEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceGroupEntity.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.orm.entities; import java.util.List; +import java.util.Objects; import javax.persistence.Column; import javax.persistence.Entity; @@ -31,6 +32,7 @@ import javax.persistence.NamedQueries; import javax.persistence.NamedQuery; import javax.persistence.OneToMany; +import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.TableGenerator; @@ -70,6 +72,13 @@ public class ServiceGroupEntity { @Column(name = "service_group_name", nullable = false, insertable = true, updatable = true) private String serviceGroupName; + /** + * Unidirectional one-to-one association to {@link StackEntity} + */ + @OneToOne + @JoinColumn(name = "stack_id", unique = false, nullable = false, insertable = true, updatable = true) + private StackEntity stack; + @ManyToOne @JoinColumn(name = "cluster_id", referencedColumnName = "cluster_id", nullable = false) private ClusterEntity clusterEntity; @@ -96,7 +105,6 @@ public void setServiceGroupId(Long serviceGroupId) { this.serviceGroupId = serviceGroupId; } - public String getServiceGroupName() { return serviceGroupName; } @@ -105,6 +113,14 @@ public void setServiceGroupName(String serviceGroupName) { this.serviceGroupName = serviceGroupName; } + public StackEntity getStack() { + return stack; + } + + public void setStack(StackEntity stack) { + this.stack = stack; + } + public List getDependencies() { return dependencies; } @@ -131,15 +147,15 @@ public boolean equals(Object o) { if (clusterId != null ? !clusterId.equals(that.clusterId) : that.clusterId != null) return false; if (serviceGroupName != null ? !serviceGroupName.equals(that.serviceGroupName) : that.serviceGroupName != null) return false; + if (Long.valueOf(stack.getStackId()) != Long.valueOf(((ServiceGroupEntity) o).getStack().getStackId())) + return false; return true; } @Override public int hashCode() { - int result = clusterId != null ? clusterId.intValue() : 0; - result = 31 * result + (serviceGroupName != null ? serviceGroupName.hashCode() : 0); - return result; + return Objects.hash(serviceGroupId, clusterId, serviceGroupName, stack); } public ClusterEntity getClusterEntity() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java index 95fa07c0b5c..635a57137fe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/StackEntity.java @@ -40,7 +40,7 @@ @TableGenerator(name = "stack_id_generator", table = "ambari_sequences", pkColumnName = "sequence_name", valueColumnName = "sequence_value", pkColumnValue = "stack_id_seq", initialValue = 0) @NamedQueries({ @NamedQuery(name = "StackEntity.findAll", query = "SELECT stack FROM StackEntity stack"), - @NamedQuery(name = "StackEntity.findByMpack", query = "SELECT stack FROM StackEntity stack where stack.currentMpackId = :currentMpackId"), + @NamedQuery(name = "StackEntity.findByMpack", query = "SELECT stack FROM StackEntity stack where stack.mpackId = :mpackId"), @NamedQuery(name = "StackEntity.findByNameAndVersion", query = "SELECT stack FROM StackEntity stack WHERE stack.stackName = :stackName AND stack.stackVersion = :stackVersion", hints = { @QueryHint(name = "eclipselink.query-results-cache", value = "true"), @@ -60,15 +60,15 @@ public class StackEntity { @Column(name = "stack_version", length = 255, nullable = false) private String stackVersion; - @Column(name = "current_mpack_id") - private Long currentMpackId; + @Column(name = "mpack_id") + private Long mpackId; - public Long getCurrentMpackId() { - return currentMpackId; + public Long getMpackId() { + return mpackId; } - public void setCurrentMpackId(Long currentMpackId) { - this.currentMpackId = currentMpackId; + public void setMpackId(Long mpackId) { + this.mpackId = mpackId; } @@ -167,7 +167,7 @@ public String toString() { buffer.append("id=").append(stackId); buffer.append(", name=").append(stackName); buffer.append(", version=").append(stackVersion); - buffer.append(", current_mpack_id=").append(currentMpackId); + buffer.append(", mpack_id=").append(mpackId); buffer.append("}"); return buffer.toString(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryAdvisor.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryAdvisor.java index 9d772ecce3b..e5246183b9d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryAdvisor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryAdvisor.java @@ -34,8 +34,12 @@ import org.apache.ambari.server.registry.RegistryRecommendationResponse.RegistryRecommendations; import org.apache.ambari.server.registry.RegistryValidationResponse.RegistryValidationResponseBuilder; import org.apache.ambari.server.registry.RegistryValidationResponse.RegistryValidationResult; +import org.apache.ambari.server.utils.MpackVersion; import org.apache.ambari.server.utils.SetUtils; + import org.apache.ambari.server.utils.VersionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.google.inject.Singleton; @@ -46,9 +50,12 @@ @Singleton public class RegistryAdvisor { + private final static Logger LOG = LoggerFactory.getLogger(RegistryAdvisor.class); + private AmbariManagementController managementController; private long requestId = 0; + /** * Given list of all possible mpack bundles, check compatibility and return compatible mpack bundles. * @param mpackBundles all possible mpack bundles @@ -76,19 +83,24 @@ private boolean isCompatibleMpackBundle(Collection mpackBundle) { } for(MpackEntry mvr: mpackMap.values()) { RegistryMpackVersion rmv = mvr.getRegistryMpackVersion(); - List compatiblities = (List) rmv.getCompatibleMpacks(); - if(compatiblities != null && !compatiblities.isEmpty()) { - for(RegistryMpackCompatiblity compatiblity : compatiblities) { - if(mpackMap.containsKey(compatiblity.getName())) { - String selectedVersion = mpackMap.get(compatiblity.getName()).getMpackVersion(); - String minVersion = compatiblity.getMinVersion(); - String maxVersion = compatiblity.getMaxVersion(); - if(minVersion != null && !minVersion.isEmpty() - && VersionUtils.compareVersions(selectedVersion, minVersion) < 0) { + List dependencies = (List) rmv.getDependencies(); + if(dependencies != null && !dependencies.isEmpty()) { + for(RegistryMpackDependency dependency : dependencies) { + if(mpackMap.containsKey(dependency.getName())) { + String selectedVersion = mpackMap.get(dependency.getName()).getMpackVersion(); + MpackVersion selectedMpackVersion = MpackVersion.parse(selectedVersion); + if(selectedMpackVersion == null) { + LOG.error("Cannot validate compatibility of mpack as the mpack version is not known"); return false; } - if(maxVersion != null && !maxVersion.isEmpty() - && VersionUtils.compareVersions(selectedVersion, maxVersion) >= 0) { + String minVersion = dependency.getMinVersion(); + String maxVersion = dependency.getMaxVersion(); + MpackVersion minMpackVersion = MpackVersion.parse(minVersion, false); + MpackVersion maxMpackVersion = MpackVersion.parse(maxVersion, false); + if(minMpackVersion != null && VersionUtils.compareTo(selectedMpackVersion, minMpackVersion) < 0) { + return false; + } + if(maxMpackVersion != null && VersionUtils.compareTo(selectedMpackVersion, maxMpackVersion) >= 0) { return false; } } @@ -248,7 +260,9 @@ public int compare(final Collection o1, final Collection for(Map.Entry mapEntry : o1Map.entrySet()) { MpackEntry o1Entry = mapEntry.getValue(); MpackEntry o2Entry = o2Map.get(mapEntry.getKey()); - int compareResult = VersionUtils.compareVersions(o1Entry.getMpackVersion(), o2Entry.getMpackVersion()); + MpackVersion v1 = MpackVersion.parse(o1Entry.getMpackVersion()); + MpackVersion v2 = MpackVersion.parse(o2Entry.getMpackVersion()); + int compareResult = VersionUtils.compareTo(v1, v2); if(compareResult > 0) { o1Wins++; } else if(compareResult < 0) { @@ -314,11 +328,15 @@ private RegistryRecommendationResponse recommendUpgradeMpacks(RegistryAdvisorReq registryMpackVersions.sort(new Comparator() { @Override public int compare(final RegistryMpackVersion o1, final RegistryMpackVersion o2) { - return -1 * VersionUtils.compareVersions(o1.getMpackVersion(), o2.getMpackVersion()); + MpackVersion v1 = MpackVersion.parse(o1.getMpackVersion()); + MpackVersion v2 = MpackVersion.parse(o2.getMpackVersion()); + return -1 * VersionUtils.compareTo(v1, v2); } }); for(RegistryMpackVersion registryMpackVersion : registryMpackVersions) { - if(VersionUtils.compareVersions(registryMpackVersion.getMpackVersion(), minMpackEntry.getMpackVersion()) > 0) { + MpackVersion version = MpackVersion.parse(registryMpackVersion.getMpackVersion()); + MpackVersion minMpackVersion = MpackVersion.parse(minMpackEntry.getMpackVersion()); + if(VersionUtils.compareTo(version, minMpackVersion) > 0) { MpackEntry mpackEntry = new MpackEntry(selectedMpackName, registryMpackVersion.getMpackVersion()); MpackBundle mpackBundle = new MpackBundle(rank++, Collections.singletonList(mpackEntry)); mpackBundles.add(mpackBundle); @@ -496,7 +514,9 @@ public int compare(final Collection o1, final Collection for(Map.Entry mapEntry : o1Map.entrySet()) { MpackEntry o1Entry = mapEntry.getValue(); MpackEntry o2Entry = o2Map.get(mapEntry.getKey()); - int compareResult = VersionUtils.compareVersions(o1Entry.getMpackVersion(), o2Entry.getMpackVersion()); + MpackVersion v1 = MpackVersion.parse(o1Entry.getMpackVersion()); + MpackVersion v2 = MpackVersion.parse(o2Entry.getMpackVersion()); + int compareResult = VersionUtils.compareTo(v1, v2); if(compareResult > 0) { o1Wins++; } else if(compareResult < 0) { @@ -514,9 +534,10 @@ public int compare(final Collection o1, final Collection for(Map.Entry selectedEntry : selectedMpacksMap.entrySet()) { String selectedKey = selectedEntry.getKey(); MpackEntry selectedMpackEntry = selectedEntry.getValue(); - if (!mpackBundleMap.containsKey(selectedKey) || - !VersionUtils.areVersionsEqual(selectedMpackEntry.getMpackVersion(), - mpackBundleMap.get(selectedKey).getMpackVersion(), true)) { + MpackVersion selectedVersion = MpackVersion.parse(selectedMpackEntry.getMpackVersion()); + MpackVersion bundleMpackVersion = MpackVersion.parse(mpackBundleMap.get(selectedKey).getMpackVersion()); + + if (!mpackBundleMap.containsKey(selectedKey) || !selectedVersion.equals(bundleMpackVersion)) { isMatch = false; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpack.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpack.java index d15a146e808..4ccd8bf921d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpack.java @@ -25,17 +25,18 @@ * Represents a single instance of a software registry */ public interface RegistryMpack { + /** - * Get mpack name + * Get mpack id * @return */ - public String getMpackName(); + public String getMpackId(); /** - * Get mpack display name + * Get mpack name * @return */ - public String getMpackDisplayName(); + public String getMpackName(); /** * Get mpack description @@ -44,10 +45,10 @@ public interface RegistryMpack { public String getMpackDescription(); /** - * Get mpack logo url + * Get mpack logo uri * @return */ - public String getMpackLogoUrl(); + public String getMpackLogoUri(); /** * Get list of mpack versions diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackCompatiblity.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackDependency.java similarity index 79% rename from ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackCompatiblity.java rename to ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackDependency.java index 2774e621913..c3f2ce2ae5f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackCompatiblity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackDependency.java @@ -20,25 +20,33 @@ import org.codehaus.jackson.annotate.JsonProperty; /** - * Represents a single instance of a software registry + * Represents registry mpack dependency */ -public interface RegistryMpackCompatiblity { +public interface RegistryMpackDependency { + + /** + * Get dependent mpack id + * @return + */ + @JsonProperty("id") + public String getId(); + /** - * Get name + * Get dependent mpack name * @return */ @JsonProperty("name") public String getName(); /** - * Get min version + * Get min version of dependent mpack * @return */ @JsonProperty("minVersion") public String getMinVersion(); /** - * Get max version + * Get max version of dependent mpack * @return */ @JsonProperty("maxVersion") diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackService.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackService.java deleted file mode 100644 index 9c3f244038a..00000000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackService.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.registry; - -import org.codehaus.jackson.annotate.JsonProperty; - -/** - * Represents a single instance of a service in a registry mpack - */ -public interface RegistryMpackService { - - /** - * Get name - * @return - */ - @JsonProperty("name") - public String getName(); - - /** - * Get version - * @return - */ - @JsonProperty("version") - public String getVersion(); - -} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackVersion.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackVersion.java index ed7abc26baa..005d2567b7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackVersion.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/RegistryMpackVersion.java @@ -19,6 +19,8 @@ import java.util.List; +import org.apache.ambari.server.state.Module; + /** * Represents a single instance of a registry mpack version */ @@ -30,39 +32,22 @@ public interface RegistryMpackVersion { public String getMpackVersion(); /** - * Get mpack build number + * Get mpack uri * @return */ - public String getMpackBuildNumber(); + public String getMpackUri(); /** - * Get mpack url + * Get mpack doc uri * @return */ - public String getMpackUrl(); + public String getMpackDocUri(); /** - * Get mpack doc url + * Get list of mpack dependencies * @return */ - public String getMpackDocUrl(); - + List getDependencies(); - /** - * Get mpack stack id - * @return - */ - public String getMpackStackId(); - - /** - * Get list of services in the mpack version - * @return - */ - public List getMpackServices(); - - /** - * Get list of compatible mpacks - * @return - */ - public List getCompatibleMpacks(); + List getModules(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpack.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpack.java index d58a29d8483..decd6953373 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpack.java @@ -32,28 +32,30 @@ */ public class JsonRegistryMpack implements RegistryMpack { + @SerializedName("id") + private String id; + @SerializedName("name") private String name; - @SerializedName("displayName") - private String displayName; - @SerializedName("description") private String description; - @SerializedName("logoUrl") - private String logoUrl; + @SerializedName("logoUri") + private String logoUri; - @SerializedName("mpackVersions") + @SerializedName("versions") private ArrayList mpackVersions; @Override - public String getMpackName() { - return name; + public String getMpackId() { + return id; } @Override - public String getMpackDisplayName() {return displayName;} + public String getMpackName() { + return name; + } @Override public String getMpackDescription() { @@ -61,8 +63,8 @@ public String getMpackDescription() { } @Override - public String getMpackLogoUrl() { - return logoUrl; + public String getMpackLogoUri() { + return logoUri; } @Override diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackCompatibility.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackDependency.java similarity index 80% rename from ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackCompatibility.java rename to ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackDependency.java index 1934b8a9ae6..e029d8ea9a7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackCompatibility.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackDependency.java @@ -17,14 +17,17 @@ */ package org.apache.ambari.server.registry.json; -import org.apache.ambari.server.registry.RegistryMpackCompatiblity; +import org.apache.ambari.server.registry.RegistryMpackDependency; import com.google.gson.annotations.SerializedName; /** - * JSON implementation of {@link RegistryMpackCompatiblity} + * JSON implementation of {@link RegistryMpackDependency} */ -public class JsonRegistryMpackCompatibility implements RegistryMpackCompatiblity { +public class JsonRegistryMpackDependency implements RegistryMpackDependency { + + @SerializedName("id") + private String id; @SerializedName("name") private String name; @@ -35,6 +38,11 @@ public class JsonRegistryMpackCompatibility implements RegistryMpackCompatiblity @SerializedName("maxVersion") private String maxVersion; + @Override + public String getId() { + return id; + } + @Override public String getName() { return name; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackService.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackService.java deleted file mode 100644 index 9ae45d59899..00000000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackService.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.registry.json; - -import org.apache.ambari.server.registry.RegistryMpackService; - -import com.google.gson.annotations.SerializedName; - -/** - * JSON implementation of {@link RegistryMpackService} - */ -public class JsonRegistryMpackService implements RegistryMpackService { - - @SerializedName("name") - private String name; - - @SerializedName("version") - private String version; - - @Override public String getName() { - return name; - } - - @Override public String getVersion() { - return version; - } -} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackVersion.java b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackVersion.java index 51aa0ca28bb..d946331fea6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackVersion.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/registry/json/JsonRegistryMpackVersion.java @@ -17,13 +17,21 @@ */ package org.apache.ambari.server.registry.json; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; import java.util.List; -import org.apache.ambari.server.registry.RegistryMpackCompatiblity; -import org.apache.ambari.server.registry.RegistryMpackService; +import org.apache.ambari.server.registry.RegistryMpackDependency; import org.apache.ambari.server.registry.RegistryMpackVersion; +import org.apache.ambari.server.state.Module; +import org.apache.ambari.server.state.Mpack; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; /** @@ -33,23 +41,16 @@ public class JsonRegistryMpackVersion implements RegistryMpackVersion { @SerializedName("version") private String version; - @SerializedName("buildNumber") - private String buildNumber; + @SerializedName("mpackUri") + private String mpackUri; - @SerializedName("mpackUrl") - private String mpackUrl; + @SerializedName("docUri") + private String docUri; - @SerializedName("docUrl") - private String docUrl; + @SerializedName("dependencies") + private ArrayList dependencies; - @SerializedName("stack-id") - private String stackId; - - @SerializedName("services") - private ArrayList services; - - @SerializedName("compatibleMpacks") - private ArrayList compatibleMpacks; + private Mpack mpack = null; @Override public String getMpackVersion() { @@ -57,32 +58,39 @@ public String getMpackVersion() { } @Override - public String getMpackBuildNumber() { - return buildNumber; + public String getMpackUri() { + return mpackUri; } @Override - public String getMpackUrl() { - return mpackUrl; + public String getMpackDocUri() { + return docUri; } @Override - public String getMpackDocUrl() { - return docUrl; + public List getDependencies() { + return dependencies; } - @Override - public List getMpackServices() { - return services; - } + private final static Logger LOG = LoggerFactory.getLogger(JsonRegistryMpackVersion.class); - @Override - public List getCompatibleMpacks() { - return compatibleMpacks; - } @Override - public String getMpackStackId() { - return stackId; + public List getModules() { + + if(mpack == null) { + try { + URL url = new URL(mpackUri); + InputStreamReader reader = new InputStreamReader(url.openStream()); + Gson gson = new Gson(); + mpack = gson.fromJson(reader, Mpack.class); + } catch (MalformedURLException e) { + LOG.warn("Failed to get list of modules, malformed mpack uri {}", mpackUri); + } + catch (IOException e) { + LOG.warn("Failed to get list of modules, mpack uri {} cannot be read", mpackUri); + } + } + return mpack == null? null: mpack.getModules(); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredRepositoryAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredRepositoryAction.java index 3c6c8e1c986..f7a2fbe030d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredRepositoryAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/UpdateDesiredRepositoryAction.java @@ -47,7 +47,7 @@ /** * Action that represents updating the Desired Stack Id during the middle of a stack upgrade (typically NonRolling). - * In a {@link org.apache.ambari.server.state.stack.upgrade.UpgradeType#NON_ROLLING}, the effective Stack Id is + * In a {@link org.apache.ambari.server.state.stack.upgrade.UpgradeType#EXPRESS}, the effective Stack Id is * actually changed half-way through calculating the Actions, and this serves to update the database to make it * evident to the user at which point it changed. */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index 8ec5c3a6dd6..79d5844c3dc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -95,7 +95,9 @@ Service addDependencyToService(String serviceGroupName, String serviceName, * @return * @throws AmbariException */ - ServiceGroup addServiceGroup(String serviceGroupName) throws AmbariException; + ServiceGroup addServiceGroup(String serviceGroupName, String version) throws AmbariException; + + ServiceGroup addServiceGroup(String serviceGroupName, StackId stackId) throws AmbariException; /** * Add service group dependency to the service group diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ModuleDependency.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ModuleDependency.java index bdce407213e..a4623910ce6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ModuleDependency.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ModuleDependency.java @@ -30,7 +30,7 @@ public enum DependencyType { @SerializedName("INSTALL") INSTALL } - @SerializedName("dependencyType") + @SerializedName("type") private DependencyType dependencyType; public String getName() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Mpack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Mpack.java index 62904f9240e..76e43d4aab6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Mpack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Mpack.java @@ -27,18 +27,25 @@ */ public class Mpack { - private Long mpackId; + /** + * Mpack DB Id + */ + private Long resourceId; private Long registryId; + /** + * Mpack id as defined in mpack.json + */ + @SerializedName("id") + private String mpackId; + @SerializedName("name") private String name; @SerializedName("version") private String version; - @SerializedName("artifacts-path") - private String artifactsPath; @SerializedName("prerequisites") private Map prerequisites; @@ -46,15 +53,6 @@ public class Mpack { @SerializedName("modules") private List modules; - @SerializedName("modules-path") - private String modulesPath; - - @SerializedName("mpack-path") - private String mpackPath; - - @SerializedName("stack-id") - private String stackId; - @SerializedName("definition") private String definition; @@ -63,12 +61,12 @@ public class Mpack { private String mpackUri; - public Long getMpackId() { - return mpackId; + public Long getResourceId() { + return resourceId; } - public void setMpackId(Long mpackId) { - this.mpackId = mpackId; + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; } public Long getRegistryId() { @@ -87,6 +85,14 @@ public void setMpackUri(String mpackUri) { this.mpackUri = mpackUri; } + public String getMpackId() { + return mpackId; + } + + public void setMpackId(String mpackId) { + this.mpackId = mpackId; + } + public String getName() { return name; } @@ -127,37 +133,6 @@ public void setModules(List modules) { this.modules = modules; } - public String getStackId() { - return stackId; - } - - public void setStackId(String stackId) { - this.stackId = stackId; - } - - public String getArtifactsPath() { - return artifactsPath; - } - - public void setArtifactsPath(String artifactsPath) { - this.artifactsPath = artifactsPath; - } - - public String getModulesPath() { - return modulesPath; - } - - public void setModulesPath(String modulesPath) { - this.modulesPath = modulesPath; - } - - public String getMpackPath() { - return mpackPath; - } - - public void setMpackPath(String mpackPath) { - this.mpackPath = mpackPath; - } public String getDefinition() { return definition; @@ -174,16 +149,13 @@ public boolean equals(Object o) { Mpack mpack = (Mpack) o; - if (!mpackId.equals(mpack.mpackId)) return false; + if (!resourceId.equals(mpack.resourceId)) return false; if (registryId != null ? !registryId.equals(mpack.registryId) : mpack.registryId != null) return false; + if (!mpackId.equals(mpack.mpackId)) return false; if (!name.equals(mpack.name)) return false; if (!version.equals(mpack.version)) return false; - if (!artifactsPath.equals(mpack.artifactsPath)) return false; if (!prerequisites.equals(mpack.prerequisites)) return false; if (!modules.equals(mpack.modules)) return false; - if (!modulesPath.equals(mpack.modulesPath)) return false; - if (!mpackPath.equals(mpack.mpackPath)) return false; - if (!stackId.equals(mpack.stackId)) return false; if (!definition.equals(mpack.definition)) return false; if (!description.equals(mpack.description)) return false; return mpackUri.equals(mpack.mpackUri); @@ -191,16 +163,13 @@ public boolean equals(Object o) { @Override public int hashCode() { - int result = mpackId.hashCode(); + int result = resourceId.hashCode(); result = 31 * result + (registryId != null ? registryId.hashCode() : 0); + result = 31 * result + mpackId.hashCode(); result = 31 * result + name.hashCode(); result = 31 * result + version.hashCode(); - result = 31 * result + artifactsPath.hashCode(); result = 31 * result + prerequisites.hashCode(); result = 31 * result + modules.hashCode(); - result = 31 * result + modulesPath.hashCode(); - result = 31 * result + mpackPath.hashCode(); - result = 31 * result + stackId.hashCode(); result = 31 * result + definition.hashCode(); result = 31 * result + description.hashCode(); result = 31 * result + mpackUri.hashCode(); @@ -210,16 +179,13 @@ public int hashCode() { @Override public String toString() { return "Mpack{" + - "mpackId=" + mpackId + + "id=" + resourceId + ", registryId=" + registryId + + ", mpackId='" + mpackId + '\'' + ", name='" + name + '\'' + ", version='" + version + '\'' + - ", artifactsPath='" + artifactsPath + '\'' + ", prerequisites=" + prerequisites + ", modules=" + modules + - ", modulesPath='" + modulesPath + '\'' + - ", mpackPath='" + mpackPath + '\'' + - ", stackId='" + stackId + '\'' + ", definition='" + definition + '\'' + ", description='" + description + '\'' + ", mpackUri='" + mpackUri + '\'' + @@ -227,6 +193,9 @@ public String toString() { } public void copyFrom(Mpack mpack) { + if (this.resourceId == null) { + this.resourceId = mpack.getResourceId(); + } if (this.name == null) { this.name = mpack.getName(); } @@ -236,9 +205,6 @@ public void copyFrom(Mpack mpack) { if (this.version == null) { this.version = mpack.getVersion(); } - if (this.stackId == null) { - this.stackId = mpack.getStackId(); - } if (this.registryId == null) { this.registryId = mpack.getRegistryId(); } @@ -248,18 +214,9 @@ public void copyFrom(Mpack mpack) { if (this.modules == null) { this.modules = mpack.getModules(); } - if (this.artifactsPath == null) { - this.artifactsPath = mpack.getArtifactsPath(); - } if (this.prerequisites == null) { this.prerequisites = mpack.getPrerequisites(); } - if (this.modulesPath == null) { - this.modulesPath = mpack.getModulesPath(); - } - if (this.mpackPath == null) { - this.mpackPath = mpack.getMpackPath(); - } if (this.definition == null) { this.definition = mpack.getDefinition(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroup.java index fcd4c893d5d..dce2337c68d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroup.java @@ -46,6 +46,8 @@ public interface ServiceGroup { Set getServiceGroupDependencyResponses(); + StackId getStackId(); + void debugDump(StringBuilder sb); void refresh(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupFactory.java index 3c5e956ec1d..59e3f40bfc1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupFactory.java @@ -29,6 +29,7 @@ public interface ServiceGroupFactory { ServiceGroup createNew(Cluster cluster, @Assisted("serviceGroupName") String serviceGroupName, + @Assisted("stackId") StackId stackId, @Assisted("serviceGroupDependencies") Set serviceGroupDependencies); ServiceGroup createExisting(Cluster cluster, ServiceGroupEntity serviceGroupEntity); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupImpl.java index dcc84337e46..5ada744eaed 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceGroupImpl.java @@ -31,10 +31,12 @@ import org.apache.ambari.server.events.publishers.AmbariEventPublisher; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.ServiceGroupDAO; +import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ServiceGroupDependencyEntity; import org.apache.ambari.server.orm.entities.ServiceGroupEntity; import org.apache.ambari.server.orm.entities.ServiceGroupEntityPK; +import org.apache.ambari.server.orm.entities.StackEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,19 +53,23 @@ public class ServiceGroupImpl implements ServiceGroup { private final Cluster cluster; private final ClusterDAO clusterDAO; + private final StackDAO stackDAO; private final ServiceGroupDAO serviceGroupDAO; private final AmbariEventPublisher eventPublisher; private final Clusters clusters; private Long serviceGroupId; private String serviceGroupName; + private StackId stackId; private Set serviceGroupDependencies; @AssistedInject public ServiceGroupImpl(@Assisted Cluster cluster, @Assisted("serviceGroupName") String serviceGroupName, + @Assisted("stackId") StackId stackId, @Assisted("serviceGroupDependencies") Set serviceGroupDependencies, ClusterDAO clusterDAO, + StackDAO stackDAO, ServiceGroupDAO serviceGroupDAO, AmbariEventPublisher eventPublisher, Clusters clusters) throws AmbariException { @@ -71,16 +77,18 @@ public ServiceGroupImpl(@Assisted Cluster cluster, this.cluster = cluster; this.clusters = clusters; this.clusterDAO = clusterDAO; + this.stackDAO = stackDAO; this.serviceGroupDAO = serviceGroupDAO; this.eventPublisher = eventPublisher; this.serviceGroupName = serviceGroupName; + this.stackId = stackId; ServiceGroupEntity serviceGroupEntity = new ServiceGroupEntity(); serviceGroupEntity.setClusterId(cluster.getClusterId()); serviceGroupEntity.setServiceGroupId(serviceGroupId); serviceGroupEntity.setServiceGroupName(serviceGroupName); - + serviceGroupEntity.setStack(stackDAO.find(stackId)); if (serviceGroupDependencies == null) { this.serviceGroupDependencies = new HashSet<>(); } else { @@ -96,17 +104,21 @@ public ServiceGroupImpl(@Assisted Cluster cluster, public ServiceGroupImpl(@Assisted Cluster cluster, @Assisted ServiceGroupEntity serviceGroupEntity, ClusterDAO clusterDAO, + StackDAO stackDAO, ServiceGroupDAO serviceGroupDAO, AmbariEventPublisher eventPublisher, Clusters clusters) throws AmbariException { this.cluster = cluster; this.clusters = clusters; this.clusterDAO = clusterDAO; + this.stackDAO = stackDAO; this.serviceGroupDAO = serviceGroupDAO; this.eventPublisher = eventPublisher; this.serviceGroupId = serviceGroupEntity.getServiceGroupId(); this.serviceGroupName = serviceGroupEntity.getServiceGroupName(); + StackEntity stack = serviceGroupEntity.getStack(); + this.stackId = new StackId(stack.getStackName(), stack.getStackVersion()); this.serviceGroupDependencies = getServiceGroupDependencies(serviceGroupEntity.getServiceGroupDependencies()); this.serviceGroupEntityPK = getServiceGroupEntityPK(serviceGroupEntity); @@ -135,6 +147,11 @@ public long getClusterId() { return cluster.getClusterId(); } + @Override + public StackId getStackId() { + return stackId; + } + @Override public Set getServiceGroupDependencies() { return serviceGroupDependencies; @@ -148,7 +165,7 @@ public void setServiceGroupDependencies(Set serviceGroupDepende @Override public ServiceGroupResponse convertToResponse() { ServiceGroupResponse r = new ServiceGroupResponse(cluster.getClusterId(), - cluster.getClusterName(), getServiceGroupId(), getServiceGroupName()); + cluster.getClusterName(), getServiceGroupId(), getServiceGroupName(), stackId.getStackId()); return r; } @@ -216,8 +233,10 @@ public Cluster getCluster() { @Override public void debugDump(StringBuilder sb) { - sb.append("ServiceGroup={ serviceGroupName=" + getServiceGroupName() + ", clusterName=" - + cluster.getClusterName() + ", clusterId=" + cluster.getClusterId() + "}"); + sb.append("ServiceGroup={ serviceGroupName=").append(getServiceGroupName()) + .append(", clusterName=").append(cluster.getClusterName()).append(", clusterId=") + .append(cluster.getClusterId()).append(", stackId=").append(getStackId()); + sb.append("}"); } /** @@ -254,10 +273,9 @@ protected ServiceGroupEntity persistEntities(ServiceGroupEntity serviceGroupEnti @Override @Transactional public void refresh() { - ServiceGroupEntityPK pk = new ServiceGroupEntityPK(); - pk.setClusterId(getClusterId()); - pk.setServiceGroupId(getServiceGroupId()); - ServiceGroupEntity serviceGroupEntity = serviceGroupDAO.findByPK(pk); + serviceGroupEntityPK.setClusterId(getClusterId()); + serviceGroupEntityPK.setServiceGroupId(getServiceGroupId()); + ServiceGroupEntity serviceGroupEntity = serviceGroupDAO.findByPK(serviceGroupEntityPK); serviceGroupDAO.refresh(serviceGroupEntity); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java index befa31bc2e3..ea1a439c251 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeContext.java @@ -74,6 +74,7 @@ import org.apache.ambari.server.state.stack.upgrade.HostOrderGrouping; import org.apache.ambari.server.state.stack.upgrade.HostOrderItem; import org.apache.ambari.server.state.stack.upgrade.HostOrderItem.HostOrderActionType; +import org.apache.ambari.server.state.stack.upgrade.Lifecycle; import org.apache.ambari.server.state.stack.upgrade.UpgradeScope; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.commons.collections.CollectionUtils; @@ -1068,7 +1069,7 @@ private UpgradeRequestValidator buildValidator(UpgradeType upgradeType){ case HOST_ORDERED: upgradeTypeValidator = new HostOrderedUpgradeValidator(); break; - case NON_ROLLING: + case EXPRESS: case ROLLING: default: upgradeTypeValidator = null; @@ -1296,7 +1297,7 @@ void check(Cluster cluster, Direction direction, UpgradeType type, UpgradePack u // verify that the upgradepack has the required grouping and set the // action items on it HostOrderGrouping hostOrderGrouping = null; - List groupings = upgradePack.getGroups(direction); + List groupings = upgradePack.getGroups(Lifecycle.LifecycleType.UPGRADE, direction); for (Grouping grouping : groupings) { if (grouping instanceof HostOrderGrouping) { hostOrderGrouping = (HostOrderGrouping) grouping; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java index 0af68400c10..94d5ba2fbca 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeHelper.java @@ -61,6 +61,7 @@ import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.Grouping; +import org.apache.ambari.server.state.stack.upgrade.Lifecycle; import org.apache.ambari.server.state.stack.upgrade.ManualTask; import org.apache.ambari.server.state.stack.upgrade.RestartTask; import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping; @@ -301,7 +302,7 @@ public List createSequence(UpgradePack upgradePack, List groups = new ArrayList<>(); UpgradeGroupHolder previousGroupHolder = null; - for (Grouping group : upgradePack.getGroups(context.getDirection())) { + for (Grouping group : upgradePack.getGroups(Lifecycle.LifecycleType.UPGRADE, context.getDirection())) { // !!! grouping is not scoped to context if (!context.isScoped(group.scope)) { @@ -337,7 +338,7 @@ public List createSequence(UpgradePack upgradePack, // NonRolling defaults to not performing service checks on a group. // Of course, a Service Check Group does indeed run them. - if (upgradePack.getType() == UpgradeType.NON_ROLLING) { + if (upgradePack.getType() == UpgradeType.EXPRESS) { group.performServiceCheck = false; } @@ -448,7 +449,7 @@ public List createSequence(UpgradePack upgradePack, StringUtils.join(hostsType.getHosts(), ','), hostsType.getMasters(), hostsType.getSecondaries()); } break; - case NON_ROLLING: + case EXPRESS: boolean isNameNodeHA = mhr.isNameNodeHA(); if (isNameNodeHA && hostsType.hasMastersAndSecondaries()) { // This could be any order, but the NameNodes have to know what role they are going to take. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index 6861816a811..9b0390faba5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -1018,13 +1018,17 @@ public void addServiceGroup(ServiceGroup serviceGroup) { } @Override - public ServiceGroup addServiceGroup(String serviceGroupName) throws AmbariException { + public ServiceGroup addServiceGroup(String serviceGroupName, String version) throws AmbariException { + return addServiceGroup(serviceGroupName, new StackId(version)); + } + + @Override + public ServiceGroup addServiceGroup(String serviceGroupName, StackId stackId) throws AmbariException { if (serviceGroups.containsKey(serviceGroupName)) { throw new AmbariException("Service group already exists" + ", clusterName=" + getClusterName() - + ", clusterId=" + getClusterId() + ", serviceGroupName=" + serviceGroupName); + + ", clusterId=" + getClusterId() + ", serviceGroupName=" + serviceGroupName); } - - ServiceGroup serviceGroup = serviceGroupFactory.createNew(this, serviceGroupName, new HashSet()); + ServiceGroup serviceGroup = serviceGroupFactory.createNew(this, serviceGroupName, stackId, new HashSet()); addServiceGroup(serviceGroup); return serviceGroup; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackMetainfoXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackMetainfoXml.java index 7f56b503c62..e26b32998e5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackMetainfoXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/StackMetainfoXml.java @@ -130,6 +130,10 @@ public Version getVersion() { public void setVersion(Version version) { this.version = version; } + + public void setOsSpecifics(List osSpecifics) { + this.osSpecifics = osSpecifics; + } /** * Internal list of os-specific details (loaded from xml). Added at schema ver 2 diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java index 7fb04f5f13b..a287cb567e9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/UpgradePack.java @@ -18,11 +18,13 @@ package org.apache.ambari.server.state.stack; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.regex.Pattern; import javax.xml.bind.Unmarshaller; @@ -35,12 +37,15 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlValue; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping; import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; import org.apache.ambari.server.state.stack.upgrade.CreateAndConfigureTask; import org.apache.ambari.server.state.stack.upgrade.Direction; import org.apache.ambari.server.state.stack.upgrade.Grouping; +import org.apache.ambari.server.state.stack.upgrade.Lifecycle; import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping; import org.apache.ambari.server.state.stack.upgrade.Task; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; @@ -55,8 +60,6 @@ @XmlAccessorType(XmlAccessType.FIELD) public class UpgradePack { - private static final String ALL_VERSIONS = "*"; - private static final Logger LOG = LoggerFactory.getLogger(UpgradePack.class); /** @@ -70,9 +73,8 @@ public class UpgradePack { @XmlElement(name="target-stack") private String targetStack; - @XmlElementWrapper(name="order") - @XmlElement(name="group") - private List groups; + @XmlElement(name="lifecycle") + public List lifecycles; @XmlElement(name="prerequisite-checks") private PrerequisiteChecks prerequisiteChecks; @@ -152,8 +154,8 @@ public UpgradeType getType() { */ public List getPrerequisiteChecks() { if (prerequisiteChecks == null) { - return new ArrayList(); - } + return new ArrayList<>(); + } return new ArrayList<>(prerequisiteChecks.checks); } @@ -164,7 +166,7 @@ public List getPrerequisiteChecks() { public PrerequisiteCheckConfig getPrerequisiteCheckConfig() { if (prerequisiteChecks == null) { return new PrerequisiteCheckConfig(); - } + } return prerequisiteChecks.configuration; } @@ -281,8 +283,16 @@ public boolean isServiceCheckFailureAutoSkipped() { return skipServiceCheckFailures; } + /** + * Used to get all groups defined for an upgrade. This method is deprecated as orchestration + * will change to per-lifecycle. At the time of writing, keep this for compilation purposes. + * @return + */ + @Experimental(feature = ExperimentalFeature.MPACK_UPGRADES) + @Deprecated public List getAllGroups() { - return groups; + + return Collections.emptyList(); } /** @@ -293,19 +303,28 @@ public List getAllGroups() { * the direction to return the ordered groups * @return the list of groups */ - public List getGroups(Direction direction) { - List list = new ArrayList<>(); + public List getGroups(Lifecycle.LifecycleType type, Direction direction) { + + // !!! lifecycles are bound to be only one per-type per-Upgrade Pack, so findFirst() is ok here + Optional optional = lifecycles.stream().filter(l -> l.type == type).findFirst(); + if (!optional.isPresent()) { + return Collections.emptyList(); + } + + List list; + List groups = optional.get().groups; + if (direction.isUpgrade()) { list = groups; } else { - switch (type) { - case NON_ROLLING: - list = getDowngradeGroupsForNonrolling(); + switch (this.type) { + case EXPRESS: + list = getDowngradeGroupsForNonrolling(groups); break; case HOST_ORDERED: case ROLLING: default: - list = getDowngradeGroupsForRolling(); + list = getDowngradeGroupsForRolling(groups); break; } } @@ -369,7 +388,7 @@ public boolean canBeApplied(String targetVersion){ * * @return the list of groups, reversed appropriately for a downgrade. */ - private List getDowngradeGroupsForRolling() { + private List getDowngradeGroupsForRolling(List groups) { List reverse = new ArrayList<>(); // !!! Testing exposed groups.size() == 1 issue. Normally there's no precedent for @@ -405,7 +424,7 @@ private List getDowngradeGroupsForRolling() { return reverse; } - private List getDowngradeGroupsForNonrolling() { + private List getDowngradeGroupsForNonrolling(List groups) { List list = new ArrayList<>(); for (Grouping g : groups) { list.add(g); @@ -477,8 +496,10 @@ private void initializeProcessingComponentMappings() { * @return {@code true} if the upgrade targets any version or stack. Both * {@link #target} and {@link #targetStack} must equal "*" */ + @Deprecated + @Experimental(feature=ExperimentalFeature.MPACK_UPGRADES) public boolean isAllTarget() { - return ALL_VERSIONS.equals(target) && ALL_VERSIONS.equals(targetStack); + return false; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Lifecycle.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Lifecycle.java new file mode 100644 index 00000000000..3d75892ff0e --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Lifecycle.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.state.stack.upgrade; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlEnum; +import javax.xml.bind.annotation.XmlEnumValue; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A lifecycle is used to delineate specific portions of an upgrade. Orchestration + * will occur based on the lifecycle phases in order they are declared in {@link LifecycleType}, + * namely: + * + *

    + *
  1. INSTALL
    2. + *
  2. QUIET
    3. + *
  3. SNAPSHOT
    4. + *
  4. PREPARE
    5. + *
  5. STOP
    6. + *
  6. UPGRADE
    7. + *
  7. START
    8. + *
  8. FINALIZE
    9. + *
+ */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement +public class Lifecycle { + + @XmlAttribute + public LifecycleType type; + + @XmlElementWrapper(name="order") + @XmlElement(name="group") + public List groups; + + /** + * Identifies the lifecycle types + */ + @XmlEnum + public enum LifecycleType { + + /** + * Work required that can be classified as installation. Normally installation of + * bits occurs outside the scope of upgrade orchestration. + */ + @XmlEnumValue("install") + INSTALL(1.0f), + + /** + * Work to stop and wait on, for example, queues or topologies. + */ + @XmlEnumValue("quiet") + QUIET(2.0f), + + /** + * Work required to snapshot or other backup. + */ + @XmlEnumValue("snapshot") + SNAPSHOT(3.0f), + + /** + * Work required to prepare to upgrade. + */ + @XmlEnumValue("prepare") + PREPARE(4.0f), + + /** + * Work required to stop a service. + */ + @XmlEnumValue("stop") + STOP(5.0f), + + /** + * For a Rolling upgrade, work required to restart and upgrade the service. + */ + @XmlEnumValue("upgrade") + UPGRADE(6.0f), + + /** + * Work required to start a service. + */ + @XmlEnumValue("start") + START(7.0f), + + /** + * Work required to finalize. Will not happen until the end of the upgrade. + */ + @XmlEnumValue("finalize") + FINALIZE(8.0f); + + private float m_order; + + private LifecycleType(float order) { + m_order = order; + } + + + /** + * Returns the ordered collection of lifecycle types. This is prefered over {@link #values()} + * to preserve ordering when adding new values. + */ + public static Collection ordered() { + return Stream.of(LifecycleType.values()).sorted((l1, l2) -> + Float.compare(l1.m_order, l2.m_order)).collect(Collectors.toList()); + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpdateStackGrouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpdateStackGrouping.java index 00998f57e76..457b7e4cfd4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpdateStackGrouping.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpdateStackGrouping.java @@ -26,7 +26,7 @@ /** * Used to represent operations that update the Stack. - * This is primarily needed during a {@link UpgradeType#NON_ROLLING} upgrade. + * This is primarily needed during a {@link UpgradeType#EXPRESS} upgrade. */ @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpgradeType.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpgradeType.java index 393c1e431bf..cbc7455f38c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpgradeType.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/UpgradeType.java @@ -28,21 +28,21 @@ public enum UpgradeType { /** * Services are up the entire time */ - @XmlEnumValue("ROLLING") + @XmlEnumValue("rolling") @SerializedName("rolling_upgrade") ROLLING, /** * All services are stopped, then started */ - @XmlEnumValue("NON_ROLLING") - @SerializedName("nonrolling_upgrade") - NON_ROLLING, + @XmlEnumValue("express") + @SerializedName("express_upgrade") + EXPRESS, /** * Host-ordered upgrade. */ - @XmlEnumValue("HOST_ORDERED") + @XmlEnumValue("host_ordered") @SerializedName("host_ordered_upgrade") HOST_ORDERED; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index f4058a1e79d..6fbb59e810a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -355,7 +355,7 @@ public void createAmbariServiceAndComponentResources(ClusterTopology topology, S } Set serviceGroupRequests = serviceGroups.stream() - .map(serviceGroupName -> new ServiceGroupRequest(clusterName, serviceGroupName)) + .map(serviceGroupName -> new ServiceGroupRequest(clusterName, serviceGroupName, Iterables.getFirst(topology.getBlueprint().getStackIds(), null).getStackId())) .collect(toSet()); Set serviceRequests = new HashSet<>(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/MpackVersion.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/MpackVersion.java index ad3fdc8e5f0..31bb1672455 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/MpackVersion.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/MpackVersion.java @@ -23,6 +23,9 @@ import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * This class should be used to compare mpack and stack versions. * Base method which should be used is parse/parseStackVersion, depends @@ -38,11 +41,15 @@ public class MpackVersion implements Comparable { private final static String VERSION_WITH_HOTFIX_AND_BUILD_PATTERN = "^([0-9]+).([0-9]+).([0-9]+)-h([0-9]+)-b([0-9]+)"; private final static String VERSION_WITH_BUILD_PATTERN = "^([0-9]+).([0-9]+).([0-9]+)-b([0-9]+)"; private final static String LEGACY_STACK_VERSION_PATTERN = "^([0-9]+).([0-9]+).([0-9]+).([0-9]+)-([0-9]+)"; + private final static String FORMAT_VERSION_PATTERN = "^([0-9]+)(\\.)?([0-9]+)?(\\.)?([0-9]+)?(-h)?([0-9]+)?(-b)?([0-9]+)?"; // Patterns for previous RE private final static Pattern PATTERN_WITH_HOTFIX = Pattern.compile(VERSION_WITH_HOTFIX_AND_BUILD_PATTERN); private final static Pattern PATTERN_LEGACY_STACK_VERSION = Pattern.compile(LEGACY_STACK_VERSION_PATTERN); private final static Pattern PATTERN_WITHOUT_HOTFIX = Pattern.compile(VERSION_WITH_BUILD_PATTERN); + private final static Pattern PATTERN_FORMAT_VERSION = Pattern.compile(FORMAT_VERSION_PATTERN); + + private final static Logger LOG = LoggerFactory.getLogger(MpackVersion.class); // Parts of version private int major; @@ -68,20 +75,59 @@ public MpackVersion(int major, int minor, int maint, int hotfix, int build) { * @return MpackVersion instance which contains parsed version * */ public static MpackVersion parse(String mpackVersion) { - Matcher versionMatcher = validateMpackVersion(mpackVersion); - MpackVersion result = null; + return parse(mpackVersion, true); + } + public static MpackVersion parse(String mpackVersion, boolean strict) { + + if(!strict) { + mpackVersion = format(mpackVersion); + } + Matcher versionMatcher = validateMpackVersion(mpackVersion); + if(versionMatcher == null) { + throw new IllegalArgumentException("Wrong format for mpack version"); + } if (versionMatcher.pattern().pattern().equals(VERSION_WITH_BUILD_PATTERN)) { - result = new MpackVersion(Integer.parseInt(versionMatcher.group(1)), Integer.parseInt(versionMatcher.group(2)), + return new MpackVersion(Integer.parseInt(versionMatcher.group(1)), Integer.parseInt(versionMatcher.group(2)), Integer.parseInt(versionMatcher.group(3)), 0, Integer.parseInt(versionMatcher.group(4))); - - } else { - result = new MpackVersion(Integer.parseInt(versionMatcher.group(1)), Integer.parseInt(versionMatcher.group(2)), + } else if (versionMatcher.pattern().pattern().equals(VERSION_WITH_HOTFIX_AND_BUILD_PATTERN)) { + return new MpackVersion(Integer.parseInt(versionMatcher.group(1)), Integer.parseInt(versionMatcher.group(2)), Integer.parseInt(versionMatcher.group(3)), Integer.parseInt(versionMatcher.group(4)), Integer.parseInt(versionMatcher.group(5))); - + } else { + throw new IllegalArgumentException("Wrong format for mpack version"); } + } - return result; + /** + * Method to format an mpack version in {major}.{minor}.{maint}-h{hotfix}-b{build} format + * @param mpackVersion input mpack version string + * @return formatted mpack version string + */ + public static String format(String mpackVersion) { + Matcher m = PATTERN_FORMAT_VERSION.matcher(mpackVersion); + if(m.matches()) { + String majorVersion = m.group(1); + String minorVersion = m.group(3); + String maintVersion = m.group(5); + String hotfixNum = m.group(7); + String buildNum = m.group(9); + if(hotfixNum != null || buildNum != null) { + if(minorVersion == null || maintVersion == null) { + // Both minorVersion and maintVersion should be specified + throw new IllegalArgumentException("Wrong format for mpack version"); + } + } + minorVersion = minorVersion != null? minorVersion: "0"; + maintVersion = maintVersion != null? maintVersion: "0"; + hotfixNum = hotfixNum != null? hotfixNum: "0"; + buildNum = buildNum != null? buildNum: "0"; + String formattedMpackVersion = String.format("%s.%s.%s-h%s-b%s", + majorVersion, minorVersion, maintVersion, hotfixNum, buildNum); + return formattedMpackVersion; + + } else { + throw new IllegalArgumentException("Wrong format for mpack version"); + } } /** @@ -93,10 +139,18 @@ public static MpackVersion parse(String mpackVersion) { * */ public static MpackVersion parseStackVersion(String stackVersion) { Matcher versionMatcher = validateStackVersion(stackVersion); - MpackVersion result = new MpackVersion(Integer.parseInt(versionMatcher.group(1)), Integer.parseInt(versionMatcher.group(2)), - Integer.parseInt(versionMatcher.group(3)), Integer.parseInt(versionMatcher.group(4)), Integer.parseInt(versionMatcher.group(5))); + if(versionMatcher == null) { + throw new IllegalArgumentException("Wrong format for mpack version"); + } - return result; + if(versionMatcher.pattern().pattern().equals(LEGACY_STACK_VERSION_PATTERN)) { + return new MpackVersion(Integer.parseInt(versionMatcher.group(1)), + Integer.parseInt(versionMatcher.group(2)), + Integer.parseInt(versionMatcher.group(3)), Integer.parseInt(versionMatcher.group(4)), + Integer.parseInt(versionMatcher.group(5))); + } else { + throw new IllegalArgumentException("Wrong format for mpack version"); + } } /** @@ -114,12 +168,9 @@ private static Matcher validateStackVersion(String version) { String stackVersion = StringUtils.trim(version); - Matcher versionMatcher = PATTERN_WITH_HOTFIX.matcher(stackVersion); + Matcher versionMatcher = PATTERN_LEGACY_STACK_VERSION.matcher(stackVersion); if (!versionMatcher.find()) { - versionMatcher = PATTERN_LEGACY_STACK_VERSION.matcher(stackVersion); - if (!versionMatcher.find()) { - throw new IllegalArgumentException("Wrong format for stack version, should be N.N.N.N-N or N.N.N-hN-bN"); - } + throw new IllegalArgumentException("Wrong format for stack version, should be N.N.N.N-N or N.N.N-hN-bN"); } return versionMatcher; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/VersionUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/VersionUtils.java index 6a3d81ce2c4..4d1e3027773 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/VersionUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/VersionUtils.java @@ -20,7 +20,10 @@ import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; + import org.apache.ambari.server.bootstrap.BootStrapImpl; + import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; @@ -253,7 +256,15 @@ public static int compareVersionsWithBuild(String version1, String version2, int compare = v2 - v1; return Integer.compare(compare, 0); - } + /** + * Helper function to compare two comparable versions with null checks + * @param v1 The first version + * @param v2 The second version + * @return 0 if both are equal, <0 if first one is lower, >0 otherwise + */ + public static int compareTo(@Nullable Comparable v1, @Nullable Comparable v2) { + return v1 == null ? (v2 == null ? 0 : -1) : v2 == null ? 1 : v1.compareTo(v2); + } } diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql index 8bec98b07da..c2f83e6755a 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql @@ -39,9 +39,9 @@ CREATE TABLE stack ( stack_id BIGINT NOT NULL, stack_name VARCHAR(255) NOT NULL, stack_version VARCHAR(255) NOT NULL, - current_mpack_id BIGINT, + mpack_id BIGINT, CONSTRAINT PK_stack PRIMARY KEY (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -123,8 +123,10 @@ CREATE TABLE servicegroups ( id BIGINT NOT NULL, service_group_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, + stack_id BIGINT NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index e7dd862b96b..c420286d3c5 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -58,9 +58,9 @@ CREATE TABLE stack ( stack_id BIGINT NOT NULL, stack_name VARCHAR(100) NOT NULL, stack_version VARCHAR(100) NOT NULL, - current_mpack_id BIGINT, + mpack_id BIGINT, CONSTRAINT PK_stack PRIMARY KEY (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -142,8 +142,10 @@ CREATE TABLE servicegroups ( id BIGINT NOT NULL, service_group_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, + stack_id BIGINT NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index c123d6cc57f..3bcba3f3717 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -39,9 +39,9 @@ CREATE TABLE stack ( stack_id NUMBER(19) NOT NULL, stack_name VARCHAR2(255) NOT NULL, stack_version VARCHAR2(255) NOT NULL, - current_mpack_id NUMBER(19), + mpack_id NUMBER(19), CONSTRAINT PK_stack PRIMARY KEY (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -123,8 +123,10 @@ CREATE TABLE servicegroups ( id NUMBER(19) NOT NULL, service_group_name VARCHAR2(255) NOT NULL, cluster_id NUMBER(19) NOT NULL, + stack_id NUMBER(19) NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id NUMBER(19) NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index bbd02397c76..b3c50070f03 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -39,9 +39,9 @@ CREATE TABLE stack ( stack_id BIGINT NOT NULL, stack_name VARCHAR(255) NOT NULL, stack_version VARCHAR(255) NOT NULL, - current_mpack_id BIGINT, + mpack_id BIGINT, CONSTRAINT PK_stack PRIMARY KEY (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -125,8 +125,10 @@ CREATE TABLE servicegroups ( id BIGINT NOT NULL, service_group_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, + stack_id BIGINT NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 2bc81748bd2..e9897b109b8 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -38,9 +38,9 @@ CREATE TABLE stack ( stack_id NUMERIC(19) NOT NULL, stack_name VARCHAR(255) NOT NULL, stack_version VARCHAR(255) NOT NULL, - current_mpack_id BIGINT, + mpack_id BIGINT, CONSTRAINT PK_stack PRIMARY KEY (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -122,8 +122,10 @@ CREATE TABLE servicegroups ( id NUMERIC(19) NOT NULL, service_group_name VARCHAR(255) NOT NULL, cluster_id NUMERIC(19) NOT NULL, + stack_id NUMERIC(19) NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id NUMBER(19) NOT NULL, diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql index ebeb17bfd4a..036ebc8a9c4 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql @@ -51,9 +51,9 @@ CREATE TABLE stack ( stack_id BIGINT NOT NULL, stack_name VARCHAR(255) NOT NULL, stack_version VARCHAR(255) NOT NULL, - current_mpack_id BIGINT, + mpack_id BIGINT, CONSTRAINT PK_stack PRIMARY KEY CLUSTERED (stack_id), - CONSTRAINT FK_mpacks FOREIGN KEY (current_mpack_id) REFERENCES mpacks(id), + CONSTRAINT FK_mpacks FOREIGN KEY (mpack_id) REFERENCES mpacks(id), CONSTRAINT UQ_stack UNIQUE (stack_name, stack_version)); CREATE TABLE extension( @@ -137,8 +137,10 @@ CREATE TABLE servicegroups ( id BIGINT NOT NULL, service_group_name VARCHAR(255) NOT NULL, cluster_id BIGINT NOT NULL, + stack_id BIGINT NOT NULL, CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), - CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); + CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id), + CONSTRAINT FK_servicegroups_stack_id FOREIGN KEY (stack_id) REFERENCES stack (stack_id)); CREATE TABLE servicegroupdependencies ( id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 2224f72170b..69cfa3e6782 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -143,7 +143,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, options = "-rollingUpgrade started" elif params.upgrade_direction == Direction.DOWNGRADE: options = "-rollingUpgrade downgrade" - elif upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: + elif upgrade_type == constants.UPGRADE_TYPE_EXPRESS: is_previous_image_dir = is_previous_fs_image() Logger.info("Previous file system image dir present is {0}".format(str(is_previous_image_dir))) @@ -209,7 +209,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, # During an Express Upgrade, NameNode will not leave SafeMode until the DataNodes are started, # so always disable the Safemode check - if upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: + if upgrade_type == constants.UPGRADE_TYPE_EXPRESS: ensure_safemode_off = False # some informative logging separate from the above logic to keep things a little cleaner diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py index 75b2eeb17e8..78db34e8a12 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode.py @@ -16,7 +16,7 @@ limitations under the License. """ -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS from resource_management.libraries.script.script import Script from resource_management.libraries.functions import stack_select @@ -64,7 +64,7 @@ def start(self, env, upgrade_type=None): def post_upgrade_restart(self, env, upgrade_type=None): # express upgrade cannot determine if the JN quorum is established - if upgrade_type == UPGRADE_TYPE_NON_ROLLING: + if upgrade_type == UPGRADE_TYPE_EXPRESS: return Logger.info("Executing Stack Upgrade post-restart") diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index 291da057223..bb32e9b9b4d 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -117,7 +117,7 @@ def start(self, env, upgrade_type=None): # after starting NN in an upgrade, touch the marker file - but only do this for certain # upgrade types - not all upgrades actually tell NN about the upgrade (like HOU) - if upgrade_type in (constants.UPGRADE_TYPE_ROLLING, constants.UPGRADE_TYPE_NON_ROLLING): + if upgrade_type in (constants.UPGRADE_TYPE_ROLLING, constants.UPGRADE_TYPE_EXPRESS): # place a file on the system indicating that we've submitting the command that # instructs NN that it is now part of an upgrade namenode_upgrade.create_upgrade_marker() @@ -200,7 +200,7 @@ def wait_for_safemode_off(self, env): def finalize_non_rolling_upgrade(self, env): hfds_binary = self.get_hdfs_binary() - namenode_upgrade.finalize_upgrade(constants.UPGRADE_TYPE_NON_ROLLING, hfds_binary) + namenode_upgrade.finalize_upgrade(constants.UPGRADE_TYPE_EXPRESS, hfds_binary) def finalize_rolling_upgrade(self, env): hfds_binary = self.get_hdfs_binary() diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py index 2094a6f4f32..61ac5a5cb2c 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka_broker.py @@ -1,4 +1,4 @@ -""" + """ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py index 29813d3f7b2..7ac07536054 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py @@ -43,7 +43,7 @@ from resource_management.core.exceptions import Fail from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook -from ambari_commons.constants import SERVICE, UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import SERVICE, UPGRADE_TYPE_EXPRESS, UPGRADE_TYPE_ROLLING from resource_management.libraries.functions.constants import Direction from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -397,7 +397,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N then copy the entire contents of that directory to the Oozie Sharelib in DFS, e.g., /usr/$stack/$current_version/atlas/hook/hive/ -> hdfs:///user/oozie/share/lib/lib_$timetamp/hive - :param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING + :param upgrade_type: If in the middle of a stack upgrade, the type as UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_EXPRESS :param upgrade_direction: If in the middle of a stack upgrade, the direction as Direction.UPGRADE or Direction.DOWNGRADE. """ import params diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py index 81497bc8bb3..18ee991e33d 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py @@ -34,7 +34,7 @@ from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS, UPGRADE_TYPE_ROLLING from oozie import oozie from oozie_service import oozie_service diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py index 2dd13baf1e2..b22e1f0d12d 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py @@ -29,7 +29,7 @@ from resource_management.core import shell from ranger_service import ranger_service from resource_management.libraries.functions import solr_cloud_util -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS from resource_management.libraries.functions.constants import Direction import setup_ranger_xml @@ -58,7 +58,7 @@ def stop(self, env, upgrade_type=None): import params env.set_params(params) - if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE: + if upgrade_type == UPGRADE_TYPE_EXPRESS and params.upgrade_direction == Direction.UPGRADE: if params.stack_supports_rolling_upgrade and not params.stack_supports_config_versioning and os.path.isfile(format('{ranger_home}/ews/stop-ranger-admin.sh')): File(format('{ranger_home}/ews/stop-ranger-admin.sh'), owner=params.unix_user, diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py index cc0075b4268..999d43b41c1 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_usersync.py @@ -26,7 +26,7 @@ from resource_management.core.logger import Logger from resource_management.core import shell from ranger_service import ranger_service -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS, UPGRADE_TYPE_ROLLING from resource_management.libraries.functions.constants import Direction import os @@ -71,7 +71,7 @@ def stop(self, env, upgrade_type=None): import params env.set_params(params) - if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE: + if upgrade_type == UPGRADE_TYPE_EXPRESS and params.upgrade_direction == Direction.UPGRADE: if params.stack_supports_usersync_non_root and os.path.isfile(params.usersync_services_file): File(params.usersync_services_file, mode = 0755 diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py index ff41cdd8531..2c1507243e7 100644 --- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/setup_ranger_xml.py @@ -34,7 +34,7 @@ from resource_management.core.utils import PasswordString from resource_management.core.shell import as_sudo from resource_management.libraries.functions import solr_cloud_util -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS, UPGRADE_TYPE_ROLLING from resource_management.core.exceptions import ExecutionFailed # This file contains functions used for setup/configure of Ranger Admin and Ranger Usersync. diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_service.py b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_service.py index 2ff48c3e974..c19781103ab 100644 --- a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_service.py +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms_service.py @@ -24,7 +24,7 @@ from resource_management.core.exceptions import ComponentIsNotRunning from resource_management.core.logger import Logger from resource_management.libraries.functions.show_logs import show_logs -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS, UPGRADE_TYPE_ROLLING from resource_management.libraries.functions.constants import Direction import os @@ -44,7 +44,7 @@ def kms_service(action='start', upgrade_type=None): show_logs(params.kms_log_dir, params.kms_user) raise elif action == 'stop': - if upgrade_type == UPGRADE_TYPE_NON_ROLLING and params.upgrade_direction == Direction.UPGRADE: + if upgrade_type == UPGRADE_TYPE_EXPRESS and params.upgrade_direction == Direction.UPGRADE: if os.path.isfile(format('{kms_home}/ranger-kms')): File(format('{kms_home}/ranger-kms'), owner=params.kms_user, diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py index 830a7ee9a86..1417a456b53 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py @@ -35,7 +35,7 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): """ :param env: Python environment :param type: Spark component type - :param upgrade_type: If in a stack upgrade, either UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_NON_ROLLING + :param upgrade_type: If in a stack upgrade, either UPGRADE_TYPE_ROLLING or UPGRADE_TYPE_EXPRESS :param action: Action to perform, such as generate configs :param config_dir: Optional config directory to write configs to. """ diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper_server.py b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper_server.py index f0e4ab919a7..d6ca2fe024e 100644 --- a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper_server.py +++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/package/scripts/zookeeper_server.py @@ -20,7 +20,7 @@ """ import random -from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING +from ambari_commons.constants import UPGRADE_TYPE_EXPRESS from resource_management.libraries.script.script import Script from resource_management.libraries.functions import get_unique_id_and_date @@ -77,7 +77,7 @@ def pre_upgrade_restart(self, env, upgrade_type=None): def post_upgrade_restart(self, env, upgrade_type=None): # during an express upgrade, there is no quorum, so don't try to perform the check - if upgrade_type == UPGRADE_TYPE_NON_ROLLING: + if upgrade_type == UPGRADE_TYPE_EXPRESS: return Logger.info("Executing Stack Upgrade post-restart") diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py index 4eecfa2b895..8d7b31ddac5 100644 --- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py @@ -27,6 +27,7 @@ from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format_jvm_option from resource_management.libraries.functions.version import format_stack_version, get_major_version +from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value from string import lower config = Script.get_config() @@ -64,7 +65,7 @@ versioned_stack_root = '/usr/hdp/current' #security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] +security_enabled = get_cluster_setting_value('security_enabled') #java params java_home = config['hostLevelParams']['java_home'] @@ -94,7 +95,7 @@ #users and groups hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] -user_group = config['configurations']['cluster-env']['user_group'] +user_group = get_cluster_setting_value('user_group') namenode_host = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_host) == 0 diff --git a/ambari-server/src/main/resources/stack-hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-ANY/scripts/params.py index e050db57ffb..09ed2e118de 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-ANY/scripts/params.py @@ -37,6 +37,7 @@ from resource_management.libraries.functions.stack_features import check_stack_feature from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions.get_architecture import get_architecture +from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value from ambari_commons.constants import AMBARI_SUDO_BINARY @@ -72,7 +73,7 @@ ambari_java_home = default("/commandParams/ambari_java_home", None) ambari_jdk_name = default("/commandParams/ambari_jdk_name", None) -security_enabled = config['configurations']['cluster-env']['security_enabled'] +security_enabled = get_cluster_setting_value('security_enabled') hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] # Some datanode settings @@ -163,7 +164,7 @@ def is_secure_port(port): #users and groups hbase_user = config['configurations']['hbase-env']['hbase_user'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] +smoke_user = get_cluster_setting_value('smokeuser') gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] gmond_user = config['configurations']['ganglia-env']["gmond_user"] tez_user = config['configurations']['tez-env']["tez_user"] @@ -173,7 +174,7 @@ def is_secure_port(port): zeppelin_user = config['configurations']['zeppelin-env']["zeppelin_user"] zeppelin_group = config['configurations']['zeppelin-env']["zeppelin_group"] -user_group = config['configurations']['cluster-env']['user_group'] +user_group = get_cluster_setting_value('user_group') ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", []) namenode_host = default("/clusterHostInfo/namenode_host", []) @@ -219,9 +220,9 @@ def is_secure_port(port): ranger_group = config['configurations']['ranger-env']['ranger_group'] dfs_cluster_administrators_group = config['configurations']['hdfs-site']["dfs.cluster.administrators"] -sysprep_skip_create_users_and_groups = default("/configurations/cluster-env/sysprep_skip_create_users_and_groups", False) -ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) -fetch_nonlocal_groups = config['configurations']['cluster-env']["fetch_nonlocal_groups"] +sysprep_skip_create_users_and_groups = get_cluster_setting_value('sysprep_skip_create_users_and_groups') +ignore_groupsusers_create = get_cluster_setting_value('ignore_groupsusers_create') +fetch_nonlocal_groups = get_cluster_setting_value('fetch_nonlocal_groups') smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") if has_hbase_masters: @@ -247,7 +248,7 @@ def is_secure_port(port): host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) tez_am_view_acls = config['configurations']['tez-site']["tez.am.view-acls"] -override_uid = str(default("/configurations/cluster-env/override_uid", "true")).lower() +override_uid = get_cluster_setting_value('override_uid') # if NN HA on secure clutser, access Zookeper securely if stack_supports_zk_security and dfs_ha_enabled and security_enabled: diff --git a/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/params.py index 50c5a40f49f..b4f48e3a6d6 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-INSTALL/scripts/params.py @@ -23,6 +23,7 @@ from resource_management.libraries.script.script import Script from resource_management.libraries.functions import default, format from resource_management.libraries.functions.expect import expect +from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -35,19 +36,19 @@ #users and groups hbase_user = config['configurations']['hbase-env']['hbase_user'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] +smoke_user = get_cluster_setting_value('smokeuser') gmetad_user = config['configurations']['ganglia-env']["gmetad_user"] gmond_user = config['configurations']['ganglia-env']["gmond_user"] tez_user = config['configurations']['tez-env']["tez_user"] -user_group = config['configurations']['cluster-env']['user_group'] +user_group = get_cluster_setting_value('user_group') proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users") hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] # repo templates -repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] -repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] +repo_rhel_suse = get_cluster_setting_value('repo_suse_rhel_template') +repo_ubuntu = get_cluster_setting_value('repo_ubuntu_template') #hosts hostname = config["hostname"] @@ -93,7 +94,7 @@ hbase_tmp_dir = "/tmp/hbase-hbase" #security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] +security_enabled = get_cluster_setting_value('security_enabled') #java params java_home = config['hostLevelParams']['java_home'] @@ -102,7 +103,7 @@ jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user jce_location = config['hostLevelParams']['jdk_location'] jdk_location = config['hostLevelParams']['jdk_location'] -ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False) +ignore_groupsusers_create = get_cluster_setting_value('ignore_groupsusers_create') host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}") diff --git a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py index 6c26e018f7c..34f85b02799 100644 --- a/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py +++ b/ambari-server/src/main/resources/stack-hooks/before-START/scripts/params.py @@ -34,6 +34,7 @@ from resource_management.libraries.functions.stack_features import get_stack_feature_version from resource_management.libraries.functions import StackFeature from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management.libraries.functions.cluster_settings import get_cluster_setting_value config = Script.get_config() tmp_dir = Script.get_tmp_dir() @@ -49,10 +50,10 @@ # Whether to skip copying fast-hdfs-resource.jar to /var/lib/ambari-agent/lib/ # This is required if tarballs are going to be copied to HDFS, so set to False -sysprep_skip_copy_fast_jar_hdfs = host_sys_prepped and default("/configurations/cluster-env/sysprep_skip_copy_fast_jar_hdfs", False) +sysprep_skip_copy_fast_jar_hdfs = host_sys_prepped and get_cluster_setting_value('sysprep_skip_copy_fast_jar_hdfs') # Whether to skip setting up the unlimited key JCE policy -sysprep_skip_setup_jce = host_sys_prepped and default("/configurations/cluster-env/sysprep_skip_setup_jce", False) +sysprep_skip_setup_jce = host_sys_prepped and get_cluster_setting_value('sysprep_skip_setup_jce') stack_version_unformatted = config['hostLevelParams']['stack_version'] stack_version_formatted = format_stack_version(stack_version_unformatted) @@ -79,7 +80,7 @@ current_service = config['serviceName'] #security params -security_enabled = config['configurations']['cluster-env']['security_enabled'] +security_enabled = get_cluster_setting_value('security_enabled') ambari_server_resources_url = default("/hostLevelParams/jdk_location", None) if ambari_server_resources_url is not None and ambari_server_resources_url.endswith('/'): @@ -98,7 +99,7 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] yarn_user = config['configurations']['yarn-env']['yarn_user'] -user_group = config['configurations']['cluster-env']['user_group'] +user_group = get_cluster_setting_value('user_group') #hosts hostname = config["hostname"] @@ -292,7 +293,7 @@ hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', None) hdfs_site = config['configurations']['hdfs-site'] -smoke_user = config['configurations']['cluster-env']['smokeuser'] +smoke_user = get_cluster_setting_value('smokeuser') smoke_hdfs_user_dir = format("/user/{smoke_user}") smoke_hdfs_user_mode = 0770 diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml deleted file mode 100644 index 2324673cd7a..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ /dev/null @@ -1,757 +0,0 @@ - - - - - - - - - - hbase-site - - - - - - - - - hbase-site - - - - - - - - - ranger-hbase-audit - - - - - - - - - - - - hbase-log4j - - - - - - - - - - - - - - - - - - - tez-site - - - - tez-site - - - - tez-site - - - - tez-site - - - - - - - - - - - - sqoop-site - - - - - - sqoop-atlas-application.properties - - - - - - - - - - - - - pig-properties - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ranger-hive-audit - - - - - - - - - - - - hive-site - - - - - - - hive-log4j - - - - - - - - hive-site - - - - - hive-site - - - - - - - - - - webhcat-site - - - - - - - - - webhcat-site - - - - - - - - - webhcat-log4j - - - - - - - - webhcat-site - - - - - - - - - - - - - ranger-env - - - - - admin-properties - - - - - - - ranger-admin-site - - - - - - - - - - - ranger-admin-site - - - - - - ranger-env - - - - - ranger-env - - - - - - - - - ranger-ugsync-site - - - - - - - - - - - ranger-kms-audit - - - - - - - - - - kms-log4j - - - - - - - - - ranger-kms-site - - - - - - - - - - - - hdfs-site - - - - - - - ranger-hdfs-audit - - - - - - - - - - - - hdfs-log4j - - - - - - - - - - - - hadoop-env - - - - - hdfs-log4j - - - - - - hadoop-env - - - - - - - - - - - - - spark-defaults - - - - - - - spark-javaopts-properties - - - - - - - - spark-defaults - - - - - spark-javaopts-properties - - - - spark-defaults - - - - - - - - spark-thrift-sparkconf - - - - - - - - - - - - - spark-javaopts-properties - - - - - - - - - - - oozie-site - - - - - oozie-log4j - - - - - - - - - - - - ranger-kafka-audit - - - - - - - - - - kafka-broker - - - - - kafka-log4j - - - - - - - - - - - - - - - - - - ranger-yarn-audit - - - - - - - - - - - yarn-log4j - - - - - - - yarn-env - - - - yarn-site - - - - yarn-env - - - - yarn-site - - - - - - - - - - - mapred-site - - - - - - - - - - - ranger-storm-audit - - - - - - - - - - - storm-site - - - - - - - - - - - - - - - - storm-site - - - - - - - storm-site - - - - - - storm-site - - - - storm-site - - - - - - storm-worker-log4j - - - - - - - - - - - - storm-cluster-log4j - - - - - - storm-worker-log4j - - - - - storm-site - - - - - - - - - - - - - - ranger-knox-audit - - - - - - - - - - gateway-log4j - - - - - - - ldap-log4j - - - - - - - - - - - - - - falcon-startup.properties - - - - falcon-log4j - - - - - - - - - - - - - - - - - - atlas-log4j - - - - - - - - - - - - - - - zookeeper-log4j - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml deleted file mode 100644 index 07b34cd9021..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml +++ /dev/null @@ -1,1103 +0,0 @@ - - - - - 2.3.*.* - HDP-2.3 - NON_ROLLING - - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - SPARK - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml deleted file mode 100644 index 331306f8bb1..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ /dev/null @@ -1,1329 +0,0 @@ - - - - - 2.4.*.* - HDP-2.4 - NON_ROLLING - - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - false - - - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - Update HBase Env Configuration - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - - - - - - Adjusting Oozie properties - - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - SPARK - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml deleted file mode 100644 index 7f9d9d2c4e4..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ /dev/null @@ -1,1478 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - NON_ROLLING - - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.ServicePresenceCheck - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Fix oozie admin users - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - false - - - - - - - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - Update HBase Env Configuration - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - Removing atlas.cluster.name property - - - - - - - Adjusting Oozie properties - - - - - - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Ranger Properties - - - - - - Configuring Ranger Alerts - - - - - - - - - - - - Adding Ranger proxy user properties - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - Update Storm log directory in storm worker log4j - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - OOZIE - SPARK - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml deleted file mode 100644 index bb22c1eddae..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ /dev/null @@ -1,1581 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - NON_ROLLING - - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.LZOCheck - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Fix oozie admin users - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - false - - - - - - - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - Adding HDFS ZKFC Security ACLs - - - - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - Validate Root Queue Ordering Policy - - - - - - Adding YARN Security ACLs - - - - - - - - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - Updating underutilized_preemption setting - - - - - - Adding service check customization property - - - - - - Modifying ATS Scan default - - - - - - - Fixing YARN Webservice Url - - - - - - Verifying LZO codec path for mapreduce - - - - - - Adding queue customization property - - - - - - - - - - - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - Update HBase Env Configuration - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - Add queue customization property - - - - - - - - - - - - - - - - - - - - - - - - - - - Adding queue customization setting - - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - - Updating hive.enforce.bucketing property to true value - - - - - - Removing atlas.cluster.name property - - - - - - - Adjusting Oozie properties - - - - - - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - Add queue customization property - - - - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Ranger Properties - - - - - - Configuring Ranger Alerts - - - - - - - - - - - - - - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - Adding Ranger proxy user properties - - - - - - - - - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - Update Storm log directory in storm worker log4j - - - - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - OOZIE - SPARK - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml deleted file mode 100644 index 19031a89c1a..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml +++ /dev/null @@ -1,907 +0,0 @@ - - - - - 2.3.*.* - HDP-2.3 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - HDP-2.2.4.2 - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - - - prepare_rolling_upgrade - - - - - - - pre_rolling_restart - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - KAFKA - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - SLIDER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Shut down all Kafka Brokers - - stop - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Adjusting Oozie properties - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml deleted file mode 100644 index 11fb0c590d9..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ /dev/null @@ -1,1018 +0,0 @@ - - - - - 2.4.*.* - HDP-2.4 - false - false - ROLLING - - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - false - - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - - - Update HBase Env Configuration - - - - - - - true - - KAFKA_BROKER - - - - - - UPGRADE - - - - - - - Verifying LZO codec path for Tez - - - - - - - UPGRADE - true - false - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - false - true - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - false - true - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - SLIDER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - Shut down all Oozie servers - - stop - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml deleted file mode 100644 index 56c1552a00f..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ /dev/null @@ -1,1236 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerAuditDbCheck - - - org.apache.ambari.server.checks.StormShutdownWarning - - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - UPGRADE - false - - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - UPGRADE - true - - - - Adding Ranger proxy user properties under kms-site - - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - true - - - - Update HBase Env Configuration - - - - - - - - UPGRADE - - - - - - - Verifying LZO codec path for Tez - - - - - - - UPGRADE - true - false - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - - UPGRADE - false - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - UPGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - DOWNGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - SLIDER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - Ranger Kerberos Keytab Check - Please follow below steps to complete upgrade in kerberos environment - -- If keytabs are managed by Ambari, Perform Regenerate Keytabs for missing components to get Ranger keytabs and restart Ranger components. - -- If keytabs are managed manually, create the required ranger keytabs in the locations specified in ranger config and restart Ranger components. - - - - - - Kafka Brokers have been upgraded with version compatability. - Config property "inter.broker.protocol.version" should be removed at your convenience. - Config property "log.message.format.version" should be removed when all clients have been upgraded. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - - - - - - Calculating Ranger Properties - - - Configuring Ranger Alerts - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - Shut down all Oozie servers - - stop - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - Fixing Oozie admin users - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml deleted file mode 100644 index faf8cbcb102..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ /dev/null @@ -1,1272 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.LZOCheck - - - org.apache.ambari.server.checks.StormShutdownWarning - - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - - UPGRADE - false - - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - UPGRADE - true - - - - Adding Ranger proxy user properties under kms-site - - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - true - - - - Update HBase Env Configuration - - - - - - - - UPGRADE - - - - - - - Verifying LZO codec path for Tez - - - - - - - UPGRADE - true - false - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - HBASE - - - YARN - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - true - - SLIDER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - - UPGRADE - false - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - UPGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - DOWNGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - Ranger Kerberos Keytab Check - Please follow below steps to complete upgrade in kerberos environment - -- If keytabs are managed by Ambari, Perform Regenerate Keytabs for missing components to get Ranger keytabs and restart Ranger components. - -- If keytabs are managed manually, create the required ranger keytabs in the locations specified in ranger config and restart Ranger components. - - - - - - Kafka Brokers have been upgraded with version compatability. - Config property "inter.broker.protocol.version" should be removed at your convenience. - Config property "log.message.format.version" should be removed when all clients have been upgraded. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - - - - - - Calculating Ranger Properties - - - Configuring Ranger Alerts - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - Fixing YARN Webservice Url - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - Validate Root Queue Ordering Policy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - Shut down all Oozie servers - - stop - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - Fixing Oozie admin users - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml deleted file mode 100644 index efb997a3c3e..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ /dev/null @@ -1,643 +0,0 @@ - - - - - - - - - - tez-site - - - - - tez-site - - - - - tez-site - - - - - - - - - - - - sqoop-site - - - - - - sqoop-atlas-application.properties - - - - - - - - - - - - - pig-properties - - - - - - - - - - - ranger-hive-audit - - - - - - - - - - - hive-site - - - - - - - hive-log4j - - - - - - - - hive-site - - - - - hive-site - - - - - - - - - webhcat-site - - - - - - - - webhcat-log4j - - - - - - - webhcat-site - - - - - - - - - - - - ranger-env - - - - - admin-properties - - - - - - - ranger-admin-site - - - - - - - - - - - ranger-admin-site - - - - - - ranger-env - - - - - ranger-env - - - - - - - - - ranger-ugsync-site - - - - - - - - - - - ranger-kms-audit - - - - - - - - - - kms-log4j - - - - - - - - - ranger-kms-site - - - - - - - - - - - - hdfs-site - - - - - - ranger-hdfs-audit - - - - - - - - - - - hdfs-log4j - - - - - - - - - - - hadoop-env - - - - hdfs-log4j - - - - - - hadoop-env - - - - - - - - - - - - ranger-yarn-audit - - - - - - - - - - - yarn-log4j - - - - - - - yarn-env - - - - yarn-site - - - - yarn-env - - - - yarn-site - - - - - - - - - yarn-site - - - - - - - - - - - - mapred-site - - - - - - - - - - - ranger-kafka-audit - - - - - - - - - - kafka-broker - - - - - kafka-log4j - - - - - - - - - - - - - - - - - - storm-site - - - - - ranger-storm-audit - - - - - - - - - - - - storm-site - - - - - - - - - - - - - - - - storm-site - - - - - - - - storm-site - - - - - storm-site - - - - - storm-worker-log4j - - - - - - - - - - - - storm-cluster-log4j - - - - - - storm-worker-log4j - - - - storm-site - - - - - - - - - - - - - ranger-hbase-audit - - - - - - - - - - - hbase-log4j - - - - - - - - - - - - - - - - - - ranger-knox-audit - - - - - - - - - - gateway-log4j - - - - - - - ldap-log4j - - - - - - - - - - - - - - falcon-startup.properties - - - - falcon-log4j - - - - - - - - - - - - - - - - - - spark-defaults - - - - - - - - spark-defaults - - - - - spark-defaults - - - - - - - - - - - - oozie-log4j - - - - - - - - - - - - - zookeeper-log4j - - - - - - - - - - - - - - atlas-log4j - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml deleted file mode 100644 index ed0d2f882c4..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml +++ /dev/null @@ -1,1092 +0,0 @@ - - - - 2.4.*.* - HDP-2.4 - NON_ROLLING - - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - Adjusting Oozie properties - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - SPARK - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml deleted file mode 100644 index 77ec871d028..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ /dev/null @@ -1,1438 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - NON_ROLLING - - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.ServicePresenceCheck - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - ATLAS_SERVER - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - Adjusting Oozie properties - - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - Fix oozie admin users - - - - - - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Ranger Properties - - - - - - - Configuring Ranger Alerts - - - - - - - - - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - Removing atlas.cluster.name property - - - - - - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - Update Storm log directory in storm worker log4j - - - - - - - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - Adding Ranger proxy user properties - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - OOZIE - SPARK - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml deleted file mode 100644 index 97824f25f7d..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ /dev/null @@ -1,1535 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - NON_ROLLING - - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.LZOCheck - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. If audit database size is too large (greater than 3GB) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - - Adding YARN Security ACLs - - - - - - - Fixing YARN Webservice Url - - - - - - - - - - - - Updating underutilized_preemption setting - - - - - - Adding service check customization property - - - - - - Modifying ATS Scan default - - - - - - Validate Root Queue Ordering Policy - - - - - - Adding queue customization property - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - Add queue customization property - - - - - - - Adjusting Oozie properties - - - - - - Fix oozie admin users - - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - Calculating Ranger Properties - - - - - - Configuring Ranger Alerts - - - - - - - - - - - - - - - - - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - Adding HDFS ZKFC Security ACLs - - - - - - - - - - - Add queue customization property - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - - Updating hive.enforce.bucketing property to true value - - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - Adding queue customization setting - - - - - - Removing atlas.cluster.name property - - - - - - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - Update Storm log directory in storm worker log4j - - - - - - - - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - Adding Ranger proxy user properties - - - - - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - UPGRADE - true - - HIVE - OOZIE - SPARK - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - SLIDER - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml deleted file mode 100644 index 80436bf305c..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml +++ /dev/null @@ -1,884 +0,0 @@ - - - - - 2.4.*.* - HDP-2.4 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - - - prepare_rolling_upgrade - - - - - - - pre_rolling_restart - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - SLIDER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Adjusting Oozie properties - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml deleted file mode 100644 index e2c7557bd33..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ /dev/null @@ -1,1187 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - false - false - ROLLING - - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerAuditDbCheck - - - org.apache.ambari.server.checks.StormShutdownWarning - - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - UPGRADE - false - - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - UPGRADE - true - - - - Adding Ranger proxy user properties under kms-site - - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - - UPGRADE - - - - - - - Verifying LZO codec path for Tez - - - - - - - UPGRADE - true - false - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - - UPGRADE - false - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - UPGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - DOWNGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - SLIDER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - Ranger Kerberos Keytab Check - Please follow below steps to complete upgrade in kerberos environment - -- If keytabs are managed by Ambari, Perform Regenerate Keytabs for missing components to get Ranger keytabs and restart Ranger components. - -- If keytabs are managed manually, create the required ranger keytabs in the locations specified in ranger config and restart Ranger components. - - - - - - Kafka Brokers have been upgraded with version compatability. - Config property "inter.broker.protocol.version" should be removed at your convenience. - Config property "log.message.format.version" should be removed when all clients have been upgraded. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - - - - - - Calculating Ranger Properties - - - Configuring Ranger Alerts - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - Shut down all Oozie servers - - stop - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - Fixing Oozie admin users - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml deleted file mode 100644 index 9e56d971d35..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ /dev/null @@ -1,1235 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - false - false - ROLLING - - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - org.apache.ambari.server.checks.ServicePresenceCheck - org.apache.ambari.server.checks.RangerAuditDbCheck - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.LZOCheck - - - org.apache.ambari.server.checks.StormShutdownWarning - - - - - - HDP-2.3.0.0 - - - Atlas - - - - - - - - - - - - UPGRADE - false - - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database. If audit database size is too large( > 3GB ) then follow the below instructions: - 1. Backup the audit table from audit database. - 2. Truncate audit table. - 3. Follow upgrade process and once completed then restore audit data to audit table. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - - - - - UPGRADE - true - - - - Adding Ranger proxy user properties under kms-site - - - - - - true - false - - RANGER_KMS_SERVER - - - - - true - - KAFKA_BROKER - - - - - - UPGRADE - - - - - - - Verifying LZO codec path for Tez - - - - - - - UPGRADE - true - false - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - stack_upgrade_save_new_config - - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - HBASE - - - YARN - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - - - - - true - - SPARK_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - true - - SLIDER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - - UPGRADE - false - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - UPGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - DOWNGRADE - false - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - Ranger Kerberos Keytab Check - Please follow below steps to complete upgrade in kerberos environment - -- If keytabs are managed by Ambari, Perform Regenerate Keytabs for missing components to get Ranger keytabs and restart Ranger components. - -- If keytabs are managed manually, create the required ranger keytabs in the locations specified in ranger config and restart Ranger components. - - - - - - Kafka Brokers have been upgraded with version compatability. - Config property "inter.broker.protocol.version" should be removed at your convenience. - Config property "log.message.format.version" should be removed when all clients have been upgraded. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - - - - - - Calculating Ranger Properties - - - Configuring Ranger Alerts - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - Fixing YARN Webservice Url - - - - - - - - - - - - - Calculating Yarn Properties for Spark Shuffle - - - - Validate Root Queue Ordering Policy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Update hive-env content - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - Shut down all Oozie servers - - stop - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - Fixing Oozie admin users - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml deleted file mode 100644 index 8c2c8605593..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ /dev/null @@ -1,615 +0,0 @@ - - - - - - - - - - storm-site - - - - storm-site - - - - - storm-worker-log4j - - - - - - - - - - - - storm-cluster-log4j - - - - - - storm-site - - - - - - storm-site - - - - - - - - - - - - - livy-conf - - - - - livy-env - - - - - - - - spark-defaults - - - - - - - - - - - spark2-log4j-properties - - - - - - - - - spark2-log4j-properties - - - - - spark2-defaults - - - - - - - - spark2-log4j-properties - - - - - - - - - - - - tez-site - - - - - - - - - - - - zookeeper-log4j - - - - - - - - - - - - - atlas-log4j - - - - - - - - application-properties - - - - - application-properties - - - - - - atlas-env - - - - application-properties - - - - - atlas-log4j - - - - - - - - - - - - oozie-log4j - - - - - - - - - - - - yarn-log4j - - - - - - - yarn-env - - - - yarn-site - - - - yarn-site - - - - yarn-env - - - - yarn-site - - - - - - - - - - - mapred-site - - - - - - - - - - - - hdfs-log4j - - - - - - - - - - - hadoop-env - - - - hdfs-log4j - - - - - - hadoop-env - - - - - - - - - - - - hbase-log4j - - - - - - - - - - - - - - - - - falcon-log4j - - - - - - - - - - - - - - - - - ranger-env - - - - admin-log4j - - - - - - - - - - usersync-log4j - - - - - - - ranger-ugsync-site - - - - - - - - tagsync-log4j - - - - - - - - - - - - kms-log4j - - - - - - - - - ranger-kms-site - - - - - - - - - - - kafka-log4j - - - - - - - - - - - - - - - - - gateway-log4j - - - - - - - ldap-log4j - - - - - - - - - - - - - - pig-properties - - - - - - - - - - - hive-env - - - - tez-site - - - - tez-site - - - - hive-log4j - - - - - - - llap-daemon-log4j - - - - - - - hive-atlas-application.properties - - - - - - - hive-site - - - - - - - - hive-interactive-env - - - - hive-interactive-env - - - - - hive-log4j2 - - - - - - - - llap-cli-log4j2 - - - - - - - - hive-interactive-env - - - - - hive-interactive-site - - - - - - hive-interactive-site - - - - - - - - tez-interactive-site - - - - - - - - - - - tez-interactive-site - - - - - hive-interactive-site - - - - - hive-site - - - - - hive-interactive-env - - - - - hive-interactive-site - - - - - hive-interactive-site - - - - - - - - - - - webhcat-log4j - - - - - - - webhcat-site - - - - - - - - - - - zeppelin-env - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/host-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/host-upgrade-2.5.xml deleted file mode 100644 index a16ef69a5ac..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/host-upgrade-2.5.xml +++ /dev/null @@ -1,593 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - false - false - HOST_ORDERED - - - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - UPGRADE - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - - - true - - - - - - - - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Adjusting Oozie properties - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml deleted file mode 100644 index 4d514aca8f3..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml +++ /dev/null @@ -1,1231 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - NON_ROLLING - - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - ATLAS_SERVER - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - ZEPPELIN_MASTER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - RANGER_TAGSYNC - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - Before continuing, please restore Zeppelin notebooks directory - - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - Verifying LZO codec path for Tez - - - - - - - Adjusting Oozie properties - - - - - - - Removing atlas.cluster.name property - - - - - - Update tez.runtime.shuffle.ssl.enable property - - - - - - - - - - - - - - - Updating Atlas TLS Exclude Protocols to exclude TLS v1.2 - - - - - - Updating Atlas zookeeper timeout values - - - - - - Updating Atlas Env gc-worker configuration - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - - false - true - - - SLIDER - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - false - - - ZEPPELIN_MASTER - - - - - false - true - false - - - ATLAS_SERVER - - - - - false - true - true - - - ATLAS_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - - UPGRADE - true - - HIVE - SPARK - SLIDER - ATLAS - ZEPPELIN - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml deleted file mode 100644 index 9caa8507671..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ /dev/null @@ -1,1639 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - NON_ROLLING - - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.LZOCheck - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - ATLAS_SERVER - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - - - - ZEPPELIN_MASTER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - RANGER_TAGSYNC - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - Before continuing, please restore Zeppelin notebooks directory - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - Adding queue customization property - - - - - - - Updating the Yarn Log4J properties to include parameterizations - - - - - - - Adding YARN Security ACLs - - - - - - - Updating underutilized_preemption setting - - - - - - Adding service check customization property - - - - - - Modifying ATS Scan default - - - - - - - Updating log aggregation retained files setting - - - - - - - Fixing YARN Webservice Url - - - - - - Validate Root Queue Ordering Policy - - - - - - - Verifying LZO codec path for Tez - - - - - - Add queue customization property - - - - - - - Adjusting Oozie properties - - - - - Updating the Oozie Log4J properties to include parameterizations - - - - - - - Updating the Hdfs Log4J properties to include parameterizations - - - - - - Adding HDFS ZKFC Security ACLs - - - - - - - Updating the Hbase Log4J properties to include parameterizations - - - - - - - Updating the Zookeeper Log4J properties to include parameterizations - - - - - - - Updating the Falcon Log4J properties to include parameterizations - - - - - - - - - - - - - - - - Updating the Ranger admin Log4J properties to include parameterizations - - - - - Updating the Ranger usersync Log4J properties to include parameterizations - - - - - Updating the Ranger tagsync Log4J properties to include parameterizations - - - - - - - - - - Updating the KMS Log4J properties to include parameterizations - - - - - - - - - - - Updating the Atlas Log4J properties to include parameterizations - - - - - - Updating Atlas TLS Exclude Protocols to exclude TLS v1.2 - - - - - - Updating Atlas zookeeper timeout values - - - - - - Updating Atlas Env gc-worker configuration - - - - - - - - - - - - - - - - - - - Updating the Kafka Log4J properties to include parameterizations - - - - - - - Updating the Knox Gateway Log4J properties to include parameterizations - - - - - Updating the Knox Ldap Log4J properties to include parameterizations - - - - - - - - - - - Updating the Storm worker Log4J properties to include parameterizations - - - - - Updating the Storm cluster Log4J properties to include parameterizations - - - - - - - - - - - - - - - Updating the Hive Log4J properties to include parameterizations - - - - - Updating the Hive llap Log4J properties to include parameterizations - - - - - Updating the Webhcat Log4J properties to include parameterizations - - - - - - Adding queue customization setting - - - - - - - - - - - - - - - - - - Updating headroom for LLAP - - - - - - Updating Hash Aggregation settings for LLAP - - - - - - Update additional LLAP settings - - - - - - Update additional LLAP-Tez settings - - - - - - Update tez.runtime.shuffle.ssl.enable property - - - - - - Updating JVM stack size for LLAP - - - - - - Updating tez shuffle parallel copies for LLAP - - - - - - - - - - Updating hive.enforce.bucketing property to true value - - - - - - - - - - - - - - Updating the LLAP Cli Log4J2 properties to include parameterizations - - - - - - Updating the Hive Log4J2 properties to include parameterizations - - - - - - Removing atlas.cluster.name property - - - - - - Updating hive atlas application properties - - - - - - - - - - - - - - - - - - - Add queue customization property - - - - - - Add queue customization property - - - - - - - - - - - - - - - - - - - - - - - - - UPGRADE - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - - false - true - - - SLIDER - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - false - true - false - - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - - - SPARK2_CLIENT - - - - - false - true - false - - - ZEPPELIN_MASTER - - - - - false - true - false - - - ATLAS_SERVER - - - - - false - true - true - - - ATLAS_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - - UPGRADE - true - - HIVE - SPARK - SPARK2 - SLIDER - ATLAS - ZEPPELIN - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - UPGRADE - true - - FALCON - KNOX - STORM - FLUME - ACCUMULO - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - configure_atlas_user_for_tagsync - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml deleted file mode 100644 index 159b81afda0..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml +++ /dev/null @@ -1,987 +0,0 @@ - - - - - 2.5.*.* - HDP-2.5 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - - - prepare_rolling_upgrade - - - - - - - pre_rolling_restart - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - - true - - KAFKA_BROKER - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - - true - - SLIDER - - - - - true - false - - HIVE_SERVER - HIVE_METASTORE - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - true - false - false - - ZEPPELIN_MASTER - - - - - true - - SPARK_CLIENT - - - - - false - true - false - - ATLAS_SERVER - - - - - false - true - true - - ATLAS_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Adjusting Oozie properties - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml deleted file mode 100644 index 4fc9b429e64..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ /dev/null @@ -1,1154 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - false - false - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - org.apache.ambari.server.checks.RangerSSLConfigCheck - org.apache.ambari.server.checks.DruidHighAvailabilityCheck - org.apache.ambari.server.checks.LZOCheck - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - - true - - KAFKA_BROKER - - - - - - - - - - - - - - - UPGRADE - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - HBASE - - - YARN - AMBARI_METRICS - LOGSEARCH - - - - - - true - - SLIDER - - - - - true - false - - HIVE_SERVER - HIVE_METASTORE - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - true - false - false - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - - - - - true - false - false - - ZEPPELIN_MASTER - - - - - true - - SPARK_CLIENT - - - - - true - - SPARK2_CLIENT - - - - - false - true - false - - ATLAS_SERVER - - - - - false - true - true - - ATLAS_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Stop Ranger Admin - - stop - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - configure_atlas_user_for_tagsync - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for mapreduce - - - - - - - - - - - - - - - - - Fixing YARN Webservice Url - - - - - - - - - - - - - - - - - - - - Validate Root Queue Ordering Policy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Verifying LZO codec path for Tez - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Adjusting Oozie properties - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Shut down all Oozie servers - - stop - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Applying Atlas proxy-user configurations. - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml deleted file mode 100644 index ca3e1421843..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ /dev/null @@ -1,293 +0,0 @@ - - - - - - - - - - zeppelin-env - - - - - - zeppelin-config - - - - - - - - - - - ranger-hdfs-audit - - - - - hadoop-env - - - - - - - - - - - - ranger-hive-audit - - - - ranger-hive-security - - - - hive-atlas-application.properties - - - - - - - - - - tez-interactive-site - - - - hive-site - - - - - - - - - - - - - - - - - - - - ranger-hbase-audit - - - - - - - - - - - ranger-knox-audit - - - - - - - - - - - ranger-storm-audit - - - - storm-site - - - - - - - - - - - - - ranger-yarn-audit - - - - yarn-site - - - - - - yarn-site - - - - - - - - - yarn-site - - - - - - - - - - yarn-site - - - - - - - - - - - ranger-kafka-audit - - - - - - - - - - - ranger-atlas-audit - - - - atlas-env - - - - application-properties - - - - atlas-log4j - - - - - atlas-env - - - - - - - - - - - ranger-kms-audit - - - - - - - - - - - spark-thrift-sparkconf - - - - - - - - - - - - spark2-thrift-sparkconf - - - - - - - - - - - - tez-site - - - - - - - - - - - - mapred-site - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml deleted file mode 100644 index a5943c18353..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/host-upgrade-2.6.xml +++ /dev/null @@ -1,607 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - false - false - HOST_ORDERED - - - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - UPGRADE - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - - - true - - - - - - - - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml deleted file mode 100644 index d3b2e569ce9..00000000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ /dev/null @@ -1,1414 +0,0 @@ - - - - - 2.6.*.* - HDP-2.6 - NON_ROLLING - - org.apache.ambari.server.checks.LZOCheck - - - - HDP-2.3.0.0 - - - - - - UPGRADE - - true - false - - - - - Before continuing, please stop all YARN queues. - - - - - - Before continuing, please stop all long-running applications deployed using Slider. E.g., su - yarn "/usr/hdp/current/slider-client/bin/slider stop <app_name>" - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - false - true - false - - - - ATLAS_SERVER - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - LIVY2_SERVER - - - - ZEPPELIN_MASTER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - DRUID_HISTORICAL - DRUID_MIDDLEMANAGER - DRUID_BROKER - DRUID_ROUTER - DRUID_COORDINATOR - DRUID_OVERLORD - DRUID_SUPERSET - - - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - PHOENIX_QUERY_SERVER - - - - KAFKA_BROKER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - NFS_GATEWAY - - - - RANGER_USERSYNC - RANGER_ADMIN - RANGER_TAGSYNC - - - - RANGER_KMS_SERVER - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s):: {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore Ranger KMS database - - - - - - Before continuing, please restore Zeppelin notebooks directory - - - - - - - - - - - - - - UPGRADE - true - - - - - - - - - - - - - - - - - - - - - - Updating hive atlas application properties - - - - - - Update tez.runtime.shuffle.ssl.enable property - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Updating the Spark classpath entries to use the specific version of Spark which is installed. - - - - - - Updating YARN NodeManager config for LLAP - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Enabling Nested Group Sync for Ranger - - - - - - - - - - - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - NFS_GATEWAY - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - RANGER_KMS_SERVER - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all YARN queues. - - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - PHOENIX_QUERY_SERVER - - - - - false - true - - - TEZ_CLIENT - - - - MAHOUT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - ZOOKEEPER - RANGER - RANGER_KMS - HDFS - KAFKA - YARN - MAPREDUCE2 - HBASE - - - - - - false - true - - - SLIDER - - - - - false - true - false - - - 1 - - - HIVE_METASTORE - HIVE_SERVER - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - false - true - false - - - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - false - true - false - - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - LIVY2_SERVER - - - - - false - true - - - SPARK_CLIENT - - - - - false - true - - - SPARK2_CLIENT - - - - - false - true - false - - - ZEPPELIN_MASTER - - - - - false - true - false - - - ATLAS_SERVER - - - - - false - true - true - - - ATLAS_CLIENT - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - - UPGRADE - true - - HIVE - SPARK - SPARK2 - SLIDER - ATLAS - ZEPPELIN - OOZIE - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - true - - DRUID_HISTORICAL - DRUID_MIDDLEMANAGER - DRUID_BROKER - DRUID_ROUTER - DRUID_COORDINATOR - DRUID_OVERLORD - DRUID_SUPERSET - - - - - UPGRADE - true - - FALCON - KNOX - STORM - FLUME - ACCUMULO - DRUID - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - set_pre_start - - - - Upgrading Ranger database schema - - setup_ranger_database - - - - - - Applying Ranger java patches - - setup_ranger_java_patches - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - Downgrading Ranger KMS database schema - - setup_ranger_kms_database - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/config-upgrade.xml deleted file mode 100644 index 9a9c9337cf2..00000000000 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/config-upgrade.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - storm-site - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/nonrolling-upgrade-2.0.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/nonrolling-upgrade-2.0.xml deleted file mode 100644 index 5d24ff8158f..00000000000 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/nonrolling-upgrade-2.0.xml +++ /dev/null @@ -1,453 +0,0 @@ - - - - - 2.0.*.* - PERF-2.0 - NON_ROLLING - - - - - HDP-2.3.0.0 - - - - - - - false - true - false - - - - FAKENODEMANAGER - FAKERESOURCEMANAGER - FAKEAPP_TIMELINE_SERVER - - - - - - - false - true - false - - - - FAKEHBASE_REGIONSERVER - FAKEHBASE_MASTER - FAKEPHOENIX_QUERY_SERVER - - - - FAKEDATANODE - FAKENAMENODE - SECONDARY_FAKENAMENODE - FAKEZKFC - FAKEJOURNALNODE - FAKENFS_GATEWAY - - - - FAKEZOOKEEPER_SERVER - - - - - - - - - - - - - - - - - - - - - - COMPLETE - true - false - - - - - actionexecute - - - - - - - false - true - false - - - FAKEZOOKEEPER_SERVER - FAKEZOOKEEPER_CLIENT - - - - - false - true - false - - - FAKEJOURNALNODE - FAKEZKFC - FAKENAMENODE - SECONDARY_FAKENAMENODE - FAKENFS_GATEWAY - FAKEHDFS_CLIENT - - - - - false - true - - - FAKEDATANODE - - - - - false - true - false - - - - FAKEHISTORYSERVER - FAKEMAPREDUCE2_CLIENT - - - - FAKEAPP_TIMELINE_SERVER - FAKERESOURCEMANAGER - FAKEYARN_CLIENT - - - - - false - true - - - - FAKENODEMANAGER - - - - - UPGRADE - false - true - - - - - - Before continuing, please start all FAKEYARN queues. - - - - - - false - true - false - - - FAKEHBASE_MASTER - FAKEHBASE_REGIONSERVER - FAKEHBASE_CLIENT - FAKEPHOENIX_QUERY_SERVER - - - - - UPGRADE - true - - FAKEZOOKEEPER - FAKEHDFS - FAKEYARN - FAKEHBASE - - - - - false - true - false - - - GRUMPY - - - - - false - true - false - - - HAPPY - - - - - UPGRADE - true - - GRUMPY - HAPPY - - - - - false - true - false - - - SLEEPY - - - - - false - true - false - - - SNOW - - - - - UPGRADE - true - - SLEEPY - SNOW - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any FAKEHBASE snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following FAKEHBASE shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/upgrade-2.0.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/upgrade-2.0.xml deleted file mode 100644 index 9c9cebbf94f..00000000000 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/upgrades/upgrade-2.0.xml +++ /dev/null @@ -1,341 +0,0 @@ - - - - - 2.0.*.* - PERF-2.0 - false - false - ROLLING - - - - - - - - - false - - FAKEZOOKEEPER_SERVER - - - - - - false - - FAKENAMENODE - FAKEJOURNALNODE - FAKEZKFC - - - - FAKEAPP_TIMELINE_SERVER - FAKERESOURCEMANAGER - - - - FAKEHBASE_MASTER - - - - - UPGRADE - true - - FAKEZOOKEEPER - FAKEHDFS - FAKEYARN - GRUMPY - HAPPY - FAKEHBASE - SLEEPY - SNOW - - - - - false - true - - FAKEDATANODE - FAKENFS_GATEWAY - - - - FAKEHBASE_REGIONSERVER - FAKEPHOENIX_QUERY_SERVER - - - - FAKENODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - FAKEZOOKEEPER - FAKEHDFS - FAKEYARN - GRUMPY - HAPPY - FAKEHBASE - SLEEPY - SNOW - - - - - false - - FAKEZOOKEEPER_CLIENT - - - - FAKEHDFS_CLIENT - - - - FAKEYARN_CLIENT - - - - FAKEHBASE_CLIENT - - - - - - UPGRADE - true - - FAKEZOOKEEPER - FAKEHDFS - FAKEYARN - GRUMPY - HAPPY - FAKEHBASE - SLEEPY - SNOW - - - - - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - UPGRADE - - You can now remove any FAKEHBASE snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following FAKEHBASE shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/main/resources/upgrade-pack.xsd b/ambari-server/src/main/resources/upgrade-pack.xsd index 249db232869..58f26f6117e 100644 --- a/ambari-server/src/main/resources/upgrade-pack.xsd +++ b/ambari-server/src/main/resources/upgrade-pack.xsd @@ -19,16 +19,29 @@ - This document describes the schema for an Upgrade Pack + This document describes the schema for an Upgrade Pack for a Management Pack + + + + + + + + + + + + + - - - + + + @@ -272,7 +285,7 @@ - + @@ -410,18 +423,6 @@ - - - - - - - - - - - - @@ -438,15 +439,21 @@ - - - - - Ensures that the element "processing" does not have duplicate services - - - - + + + + + + + Ensures that the element "processing" does not have duplicate services + + + + + + + + @@ -466,6 +473,12 @@ + + + + + + diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java index 9bea4aeec88..50653eb9888 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java @@ -116,7 +116,7 @@ public static void setup() throws AmbariException { OrmTestHelper helper = injector.getInstance(OrmTestHelper.class); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster1); - ServiceGroup serviceGroup = cluster1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster1.addServiceGroup("CORE", "HDP-2.6.0"); cluster1.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); SERVICE_SITE_CLUSTER = new HashMap<>(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionSchedulerThreading.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionSchedulerThreading.java index 6688510a5c8..64ce0759982 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionSchedulerThreading.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionSchedulerThreading.java @@ -109,7 +109,7 @@ public void testDesiredConfigurationsAfterApplyingLatestForStackInOtherThreads() // add a service String serviceName = "ZOOKEEPER"; RepositoryVersionEntity repositoryVersion = ormTestHelper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service service = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); String configType = "zoo.cfg"; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java index bc5a50a3a08..a23fae966a4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/HeartbeatProcessorTest.java @@ -1311,7 +1311,7 @@ public void testComponentInProgressStatusSafeAfterStatusReport() throws Exceptio */ private Service addService(Cluster cluster, String serviceName) throws AmbariException { RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0"); return cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java index 42374c4109d..2a7b8b2784b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java @@ -1604,7 +1604,7 @@ private File createTestKeytabData(HeartBeatHandler heartbeatHandler) throws Exce */ private Service addService(Cluster cluster, String serviceName) throws AmbariException { RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0"); return cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java index 69546bfdc46..5c96ece24ef 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatMonitor.java @@ -174,7 +174,7 @@ public void testStateCommandsGeneration() throws AmbariException, InterruptedExc clusters.mapAndPublishHostsToCluster(hostNames, clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); hdfs.addServiceComponent(Role.DATANODE.name()); hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost(hostname1); @@ -277,7 +277,7 @@ public void testStatusCommandForAnyComponents() throws Exception { clusters.mapAndPublishHostsToCluster(hostNames, clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); hdfs.addServiceComponent(Role.DATANODE.name()); hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost @@ -385,7 +385,7 @@ public void testHeartbeatStateCommandsEnqueueing() throws AmbariException, Inter clusters.mapAndPublishHostsToCluster(hostNames, clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); hdfs.addServiceComponent(Role.DATANODE.name()); hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost(hostname1); @@ -467,7 +467,7 @@ public void testHeartbeatLossWithComponent() throws AmbariException, Interrupted clusters.mapAndPublishHostsToCluster(hostNames, clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); hdfs.addServiceComponent(Role.DATANODE.name()); hdfs.getServiceComponent(Role.DATANODE.name()).addServiceComponentHost(hostname1); @@ -586,7 +586,7 @@ public void testStateCommandsWithAlertsGeneration() throws AmbariException, Inte clusters.mapAndPublishHostsToCluster(hostNames, clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, serviceName, serviceName, repositoryVersion); hdfs.addServiceComponent(Role.DATANODE.name()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/resources/HostResourceDefinitionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/resources/HostResourceDefinitionTest.java index c028749a2be..f6dbd58d48d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/resources/HostResourceDefinitionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/resources/HostResourceDefinitionTest.java @@ -19,10 +19,13 @@ package org.apache.ambari.server.api.resources; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import java.util.Set; +import org.apache.ambari.server.api.query.render.DefaultRenderer; +import org.apache.ambari.server.api.query.render.HostSummaryRenderer; import org.apache.ambari.server.controller.spi.Resource; import org.junit.Test; @@ -53,6 +56,13 @@ public void testGetSubResourceDefinitions() { assertTrue(includesType(subResources, Resource.Type.HostKerberosIdentity)); } + @Test + public void getRenderer() { + ResourceDefinition resource = new HostResourceDefinition(); + assertSame(HostSummaryRenderer.class, resource.getRenderer("summary").getClass()); + assertSame(DefaultRenderer.class, resource.getRenderer(null).getClass()); + } + private boolean includesType(Set resources, Resource.Type type) { for (SubResourceDefinition subResource : resources) { if (subResource.getType() == type) { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java index 8b185ca0b31..1b2ffb35200 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java @@ -1741,12 +1741,12 @@ public void testAlertDefinitionMerging() throws Exception { Clusters clusters = injector.getInstance(Clusters.class); Cluster cluster = clusters.getClusterById(clusterId); - cluster.setDesiredStackVersion( - new StackId(STACK_NAME_HDP, stackVersion)); + StackId stackId = new StackId(STACK_NAME_HDP, stackVersion); + cluster.setDesiredStackVersion(stackId); RepositoryVersionEntity repositoryVersion = ormHelper.getOrCreateRepositoryVersion( cluster.getCurrentStackVersion(), repoVersion); - ServiceGroup sg = cluster.addServiceGroup("core"); + ServiceGroup sg = cluster.addServiceGroup("core", stackId.getStackId()); cluster.addService(sg, "HDFS", "HDFS", repositoryVersion); metaInfo.reconcileAlertDefinitions(clusters, false); @@ -1861,7 +1861,7 @@ public void testAlertDefinitionMergingRemoveScenario() throws Exception { RepositoryVersionEntity repositoryVersion = ormHelper.getOrCreateRepositoryVersion( cluster.getCurrentStackVersion(), repoVersion); - ServiceGroup sg = cluster.addServiceGroup("core"); + ServiceGroup sg = cluster.addServiceGroup("core", "HDP-2.0.6"); cluster.addService(sg, "HDFS", "HDFS", repositoryVersion); metaInfo.reconcileAlertDefinitions(clusters, false); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java index 991534651d0..cb973c55d6d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java @@ -207,23 +207,23 @@ public void testIsApplicableForPatch() throws Exception { public void testRequired() throws Exception { RollingTestCheckImpl rollingCheck = new RollingTestCheckImpl(PrereqCheckType.SERVICE); Assert.assertTrue(rollingCheck.isRequired(UpgradeType.ROLLING)); - Assert.assertFalse(rollingCheck.isRequired(UpgradeType.NON_ROLLING)); + Assert.assertFalse(rollingCheck.isRequired(UpgradeType.EXPRESS)); NotRequiredCheckTest notRequiredCheck = new NotRequiredCheckTest(PrereqCheckType.SERVICE); Assert.assertFalse(notRequiredCheck.isRequired(UpgradeType.ROLLING)); - Assert.assertFalse(notRequiredCheck.isRequired(UpgradeType.NON_ROLLING)); + Assert.assertFalse(notRequiredCheck.isRequired(UpgradeType.EXPRESS)); Assert.assertFalse(notRequiredCheck.isRequired(UpgradeType.HOST_ORDERED)); TestCheckImpl requiredCheck = new TestCheckImpl(PrereqCheckType.SERVICE); Assert.assertTrue(requiredCheck.isRequired(UpgradeType.ROLLING)); - Assert.assertTrue(requiredCheck.isRequired(UpgradeType.NON_ROLLING)); + Assert.assertTrue(requiredCheck.isRequired(UpgradeType.EXPRESS)); Assert.assertTrue(requiredCheck.isRequired(UpgradeType.HOST_ORDERED)); } @UpgradeCheck( group = UpgradeCheckGroup.DEFAULT, order = 1.0f, - required = { UpgradeType.ROLLING, UpgradeType.NON_ROLLING, UpgradeType.HOST_ORDERED }) + required = { UpgradeType.ROLLING, UpgradeType.EXPRESS, UpgradeType.HOST_ORDERED }) private class TestCheckImpl extends AbstractCheckDescriptor { private PrereqCheckType m_type; private Set m_applicableServices = Sets.newHashSet(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/configuration/RecoveryConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/configuration/RecoveryConfigHelperTest.java index 06d773e6782..f0fff2b19d7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/configuration/RecoveryConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/configuration/RecoveryConfigHelperTest.java @@ -148,7 +148,7 @@ public void testServiceComponentInstalled() Cluster cluster = heartbeatTestHelper.getDummyCluster(); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); @@ -183,7 +183,7 @@ public void testServiceComponentUninstalled() throws Exception { Cluster cluster = heartbeatTestHelper.getDummyCluster(); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); @@ -220,7 +220,7 @@ public void testClusterEnvConfigChanged() throws Exception { Cluster cluster = heartbeatTestHelper.getDummyCluster(); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); @@ -261,7 +261,7 @@ public void testMaintenanceModeChanged() throws Exception { Cluster cluster = heartbeatTestHelper.getDummyCluster(); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); @@ -297,7 +297,7 @@ public void testServiceComponentRecoveryChanged() throws Exception { Cluster cluster = heartbeatTestHelper.getDummyCluster(); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); @@ -341,7 +341,7 @@ public void testMultiNodeCluster() RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(cluster); // Add HDFS service with DATANODE component to the cluster - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.6.0.0"); Service hdfs = cluster.addService(serviceGroup, HDFS, HDFS, repositoryVersion); hdfs.addServiceComponent(DATANODE).setRecoveryEnabled(true); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java index b0ac089f334..8fb34c00296 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java @@ -786,7 +786,7 @@ private void createClusterFixture(String clusterName, StackId stackId, Assert.assertNotNull(cluster); String serviceGroupName = "CORE"; - cluster.addServiceGroup(serviceGroupName); + cluster.addServiceGroup(serviceGroupName, stackId.getStackId()); createService(clusterName, serviceGroupName, "HDFS", repositoryVersion); createService(clusterName, serviceGroupName, "YARN", repositoryVersion); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index 85686e25c52..4b996e573b2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -2225,7 +2225,7 @@ public void testRegisterMpacks() throws Exception{ MpackRequest mpackRequest = createNiceMock(MpackRequest.class); RequestStatusResponse response = new RequestStatusResponse(new Long(201)); Mpack mpack = new Mpack(); - mpack.setMpackId((long)100); + mpack.setResourceId((long)100); mpack.setModules(new ArrayList()); mpack.setPrerequisites(new HashMap()); mpack.setRegistryId(new Long(100)); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index b09f1528ac9..1b4033b1938 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1043,7 +1043,7 @@ public void testCreateServiceComponentWithInvalidRequest() RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); - ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName, stackId.getStackId()); Service s1 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "MAPREDUCE", "MAPREDUCE", repositoryVersion); c1.addService(s1); @@ -1364,7 +1364,7 @@ public void testCreateServiceComponentMultiple() throws Exception { RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName, stackId.getStackId()); Service s1 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "MAPREDUCE", "MAPREDUCE", repositoryVersion); c1.addService(s1); @@ -1625,11 +1625,11 @@ public void testCreateServiceComponentHostWithInvalidRequest() throws Exception // Expected } - Service s1 = serviceFactory.createNew(foo, foo.addServiceGroup(serviceGroupName), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); + Service s1 = serviceFactory.createNew(foo, foo.addServiceGroup(serviceGroupName, stackId.getStackId()), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); foo.addService(s1); - Service s2 = serviceFactory.createNew(c1, c1.addServiceGroup(serviceGroupName), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); + Service s2 = serviceFactory.createNew(c1, c1.addServiceGroup(serviceGroupName, stackId.getStackId()), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); c1.addService(s2); - Service s3 = serviceFactory.createNew(c2, c2.addServiceGroup(serviceGroupName), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); + Service s3 = serviceFactory.createNew(c2, c2.addServiceGroup(serviceGroupName, stackId.getStackId()), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); c2.addService(s3); @@ -2215,7 +2215,7 @@ public void testGetServices() throws Exception { clusters.addCluster(cluster1, stackId); Cluster c1 = clusters.getCluster(cluster1); - ServiceGroup serviceGroup = c1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c1.addServiceGroup("CORE", stackId.getStackId()); Service s1 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); c1.addService(s1); @@ -2251,11 +2251,11 @@ public void testGetServicesWithFilters() throws Exception { c1.setDesiredStackVersion(stackId); c2.setDesiredStackVersion(stackId); - ServiceGroup sg1 = c1.addServiceGroup("CORE"); + ServiceGroup sg1 = c1.addServiceGroup("CORE", stackId.getStackId()); Service s1 = serviceFactory.createNew(c1, sg1, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(c1, sg1, new ArrayList<>(),"MAPREDUCE", "MAPREDUCE", repositoryVersion); Service s3 = serviceFactory.createNew(c1, sg1, new ArrayList<>(),"HBASE", "HBASE", repositoryVersion); - ServiceGroup sg2 = c2.addServiceGroup("CORE"); + ServiceGroup sg2 = c2.addServiceGroup("CORE", stackId.getStackId()); Service s4 = serviceFactory.createNew(c2, sg2, new ArrayList<>(),"HIVE", "HIVE", repositoryVersion); Service s5 = serviceFactory.createNew(c2, sg2, new ArrayList<>(),"ZOOKEEPER", "ZOOKEEPER", repositoryVersion); @@ -2329,7 +2329,7 @@ public void testGetServiceComponents() throws Exception { Cluster c1 = clusters.getCluster(cluster1); c1.setDesiredStackVersion(stackId); String serviceGroupName = "CORE"; - Service s1 = serviceFactory.createNew(c1, c1.addServiceGroup(serviceGroupName), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); + Service s1 = serviceFactory.createNew(c1, c1.addServiceGroup(serviceGroupName, "HDP-0.2"), new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); c1.addService(s1); s1.setDesiredState(State.INSTALLED); ServiceComponent sc1 = serviceComponentFactory.createNew(s1, "DATANODE"); @@ -2369,11 +2369,11 @@ public void testGetServiceComponentsWithFilters() throws Exception { Cluster c1 = clusters.getCluster(cluster1); Cluster c2 = clusters.getCluster(cluster2); - ServiceGroup sg1 = c1.addServiceGroup("CORE"); + ServiceGroup sg1 = c1.addServiceGroup("CORE", stackId.getStackId()); Service s1 = serviceFactory.createNew(c1, sg1, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(c1, sg1, new ArrayList<>(), "MAPREDUCE", "MAPREDUCE", repositoryVersion); Service s3 = serviceFactory.createNew(c1, sg1, new ArrayList<>(), "HBASE", "HBASE", repositoryVersion); - ServiceGroup sg2 = c2.addServiceGroup("CORE"); + ServiceGroup sg2 = c2.addServiceGroup("CORE", stackId.getStackId()); Service s4 = serviceFactory.createNew(c2, sg2, new ArrayList<>(), "HIVE", "HIVE", repositoryVersion); Service s5 = serviceFactory.createNew(c2, sg2, new ArrayList<>(), "ZOOKEEPER", "ZOOKEEPER", repositoryVersion); @@ -2486,7 +2486,7 @@ public void testGetServiceComponentHosts() throws Exception { Cluster c1 = setupClusterWithHosts(cluster1, "HDP-0.1", Lists.newArrayList(host1), "centos5"); RepositoryVersionEntity repositoryVersion = repositoryVersion01; - ServiceGroup serviceGroup = c1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c1.addServiceGroup("CORE", "HDP-0.1"); Service s1 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); c1.addService(s1); ServiceComponent sc1 = serviceComponentFactory.createNew(s1, "DATANODE"); @@ -2858,7 +2858,7 @@ public void testGetServiceComponentHostsWithFilters() throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion02; String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName, "HDP-0.2"); Service s1 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "MAPREDUCE", "MAPREDUCE", repositoryVersion); Service s3 = serviceFactory.createNew(c1, serviceGroup, new ArrayList<>(), "HBASE", "HBASE", repositoryVersion); @@ -3849,7 +3849,7 @@ public void testCreateCustomActions() throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion206; String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName, "HDP-2.0.6"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); Service mapred = cluster.addService(serviceGroup, "YARN", "YARN", repositoryVersion); @@ -4007,7 +4007,7 @@ public void testComponentCategorySentWithRestart() throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion207; String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName, "HDP-2.0.7"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); hdfs.addServiceComponent(Role.HDFS_CLIENT.name()); @@ -4113,7 +4113,7 @@ public void testCreateActionsFailures() throws Exception { cluster.addDesiredConfig("_test", Collections.singleton(config2)); String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = cluster.addServiceGroup(serviceGroupName, "HDP-2.0.7"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); Service hive = cluster.addService(serviceGroup, "HIVE", "HIVE", repositoryVersion); @@ -4381,7 +4381,7 @@ public void testCreateServiceCheckActions() throws Exception { cluster.addDesiredConfig("_test", Collections.singleton(config1)); cluster.addDesiredConfig("_test", Collections.singleton(config2)); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-0.1"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); Service mapReduce = cluster.addService(serviceGroup, "MAPREDUCE", "MAPREDUCE", repositoryVersion); @@ -5916,7 +5916,7 @@ public void testResourceFiltersWithCustomActions() throws Exception { put("key1", "value1"); }}, new HashMap<>()); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.0.6"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); Service mapred = cluster.addService(serviceGroup, "YARN", "YARN", repositoryVersion); @@ -6014,7 +6014,7 @@ public void testResourceFiltersWithCustomCommands() throws Exception { put("key1", "value1"); }}, new HashMap<>()); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-2.0.6"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); Service mapred = cluster.addService(serviceGroup, "YARN", "YARN", repositoryVersion); @@ -6708,56 +6708,6 @@ public void testGetStacks() throws Exception { } } - @Test - public void testGetStackVersions() throws Exception { - - - StackVersionRequest request = new StackVersionRequest(STACK_NAME, null); - Set responses = controller.getStackVersions(Collections.singleton(request)); - Assert.assertEquals(STACK_VERSIONS_CNT, responses.size()); - - StackVersionRequest requestWithParams = new StackVersionRequest(STACK_NAME, STACK_VERSION); - Set responsesWithParams = controller.getStackVersions(Collections.singleton(requestWithParams)); - Assert.assertEquals(1, responsesWithParams.size()); - for (StackVersionResponse responseWithParams: responsesWithParams) { - Assert.assertEquals(responseWithParams.getStackVersion(), STACK_VERSION); - } - - StackVersionRequest invalidRequest = new StackVersionRequest(STACK_NAME, NON_EXT_VALUE); - try { - controller.getStackVersions(Collections.singleton(invalidRequest)); - } catch (StackAccessException e) { - // do nothing - } - - // test that a stack response has upgrade packs - requestWithParams = new StackVersionRequest(STACK_NAME, "2.1.1"); - responsesWithParams = controller.getStackVersions(Collections.singleton(requestWithParams)); - - Assert.assertEquals(1, responsesWithParams.size()); - StackVersionResponse resp = responsesWithParams.iterator().next(); - assertNotNull(resp.getUpgradePacks()); - assertTrue(resp.getUpgradePacks().size() > 0); - assertTrue(resp.getUpgradePacks().contains("upgrade_test")); - } - - @Test - public void testGetStackVersionActiveAttr() throws Exception { - - for (StackInfo stackInfo: ambariMetaInfo.getStacks(STACK_NAME)) { - if (stackInfo.getVersion().equalsIgnoreCase(STACK_VERSION)) { - stackInfo.setActive(true); - } - } - - StackVersionRequest requestWithParams = new StackVersionRequest(STACK_NAME, STACK_VERSION); - Set responsesWithParams = controller.getStackVersions(Collections.singleton(requestWithParams)); - Assert.assertEquals(1, responsesWithParams.size()); - for (StackVersionResponse responseWithParams: responsesWithParams) { - Assert.assertTrue(responseWithParams.isActive()); - } - } - @Test public void testGetRepositories() throws Exception { @@ -8932,7 +8882,7 @@ public void testReinstallClientSchSkippedInMaintenance() throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion120; String serviceGroupName = "CORE"; - ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName); + ServiceGroup serviceGroup = c1.addServiceGroup(serviceGroupName, "HDP-1.2.0"); Service hdfs = c1.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); createServiceComponent(cluster1, serviceGroupName, "HDFS", "NAMENODE", State.INIT); createServiceComponent(cluster1, serviceGroupName, "HDFS", "DATANODE", State.INIT); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackRequestTest.java index 694c499508e..6c48906d789 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackRequestTest.java @@ -29,7 +29,7 @@ public class MpackRequestTest { public void testBasicGetAndSet() { MpackRequest mpackRequest = new MpackRequest(1L); - Assert.assertEquals((Long)1L, mpackRequest.getMpackId()); + Assert.assertEquals((Long)1L, mpackRequest.getId()); mpackRequest.setMpackUri("abc.tar.gz"); mpackRequest.setRegistryId(1L); mpackRequest.setMpackVersion("3.0"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackResponseTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackResponseTest.java index 9273a04ca70..0b60e109d49 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackResponseTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/MpackResponseTest.java @@ -32,7 +32,7 @@ public class MpackResponseTest { @Test public void testBasicGetAndSet() { MpackResponse mpackResponse = new MpackResponse(setupMpack()); - Assert.assertEquals((Long)100L, mpackResponse.getMpackId()); + Assert.assertEquals((Long)100L, mpackResponse.getId()); Assert.assertEquals((Long)100L, mpackResponse.getRegistryId()); Assert.assertEquals("3.0",mpackResponse.getMpackVersion()); Assert.assertEquals("abc.tar.gz",mpackResponse.getMpackUri()); @@ -41,7 +41,7 @@ public void testBasicGetAndSet() { } public Mpack setupMpack() { Mpack mpack = new Mpack(); - mpack.setMpackId(100L); + mpack.setResourceId(100L); mpack.setModules(new ArrayList()); mpack.setPrerequisites(new HashMap()); mpack.setRegistryId(100L); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/RegistryRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/RegistryRequestTest.java index c61cafb3ade..fb107604f98 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/RegistryRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/RegistryRequestTest.java @@ -28,9 +28,9 @@ public class RegistryRequestTest { @Test public void testBasicGetAndSet() { RegistryRequest registryRequest = - new RegistryRequest((Long) 1L, "hwx", RegistryType.JSON, "abc.tar.gz"); + new RegistryRequest((Long) 1L, "hwx", RegistryType.JSON, "hwx-software-registry.json"); Assert.assertEquals((Long) 1L, registryRequest.getRegistryId()); - Assert.assertEquals("abc.tar.gz", registryRequest.getRegistryUri()); + Assert.assertEquals("hwx-software-registry.json", registryRequest.getRegistryUri()); Assert.assertEquals("hwx", registryRequest.getRegistryName()); Assert.assertEquals(RegistryType.JSON, registryRequest.getRegistryType()); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CompatibleRepositoryVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CompatibleRepositoryVersionResourceProviderTest.java index 7df094298d2..3269cb02a53 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CompatibleRepositoryVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/CompatibleRepositoryVersionResourceProviderTest.java @@ -161,7 +161,7 @@ public String getTargetStack() { @Override public UpgradeType getType() { - return UpgradeType.NON_ROLLING; + return UpgradeType.EXPRESS; } }; @@ -212,7 +212,7 @@ public String getTarget() { @Override public UpgradeType getType() { - return UpgradeType.NON_ROLLING; + return UpgradeType.EXPRESS; } }; @@ -300,7 +300,7 @@ public void testGetResources() throws Exception { // Test For Upgrade Types Map> versionToUpgradeTypesMap = new HashMap<>(); versionToUpgradeTypesMap.put("1.1", Arrays.asList(UpgradeType.ROLLING)); - versionToUpgradeTypesMap.put("2.2", Arrays.asList(UpgradeType.NON_ROLLING, UpgradeType.ROLLING)); + versionToUpgradeTypesMap.put("2.2", Arrays.asList(UpgradeType.EXPRESS, UpgradeType.ROLLING)); assertEquals(versionToUpgradeTypesMap.size(), checkUpgradeTypes(resources, versionToUpgradeTypesMap)); // !!! verify we can get services @@ -382,7 +382,7 @@ public void testGetResourcesWithAmendedPredicate() throws Exception { // Test For Upgrade Types Map> versionToUpgradeTypesMap = new HashMap<>(); versionToUpgradeTypesMap.put("1.1", Arrays.asList(UpgradeType.ROLLING)); - versionToUpgradeTypesMap.put("2.2", Arrays.asList(UpgradeType.NON_ROLLING, UpgradeType.ROLLING)); + versionToUpgradeTypesMap.put("2.2", Arrays.asList(UpgradeType.EXPRESS, UpgradeType.ROLLING)); assertEquals(versionToUpgradeTypesMap.size(), checkUpgradeTypes(resources, versionToUpgradeTypesMap)); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/MpackResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/MpackResourceProviderTest.java index 6faec3a200d..bb1dbec4ef3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/MpackResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/MpackResourceProviderTest.java @@ -76,14 +76,14 @@ public void testGetResourcesMpacks() throws Exception { Resource.Type type = Resource.Type.Mpack; Resource resourceExpected1 = new ResourceImpl(Resource.Type.Mpack); - resourceExpected1.setProperty(MpackResourceProvider.MPACK_ID, (long)1); + resourceExpected1.setProperty(MpackResourceProvider.MPACK_RESOURCE_ID, (long)1); resourceExpected1.setProperty(MpackResourceProvider.MPACK_NAME, "TestMpack1"); resourceExpected1.setProperty(MpackResourceProvider.MPACK_VERSION, "3.0"); resourceExpected1.setProperty(MpackResourceProvider.MPACK_URI, "abcd.tar.gz"); resourceExpected1.setProperty(MpackResourceProvider.REGISTRY_ID, null); Resource resourceExpected2 = new ResourceImpl(Resource.Type.Mpack); - resourceExpected2.setProperty(MpackResourceProvider.MPACK_ID, (long)2); + resourceExpected2.setProperty(MpackResourceProvider.MPACK_RESOURCE_ID, (long)2); resourceExpected2.setProperty(MpackResourceProvider.MPACK_NAME, "TestMpack2"); resourceExpected2.setProperty(MpackResourceProvider.MPACK_VERSION, "3.0"); resourceExpected2.setProperty(MpackResourceProvider.MPACK_URI, "abc.tar.gz"); @@ -91,14 +91,14 @@ public void testGetResourcesMpacks() throws Exception { List entities = new ArrayList<>(); MpackEntity entity = new MpackEntity(); - entity.setMpackId((long) 1); + entity.setId((long) 1); entity.setMpackUri("abcd.tar.gz"); entity.setMpackName("TestMpack1"); entity.setMpackVersion("3.0"); entities.add(entity); entity = new MpackEntity(); - entity.setMpackId((long) 2); + entity.setId((long) 2); entity.setMpackUri("abc.tar.gz"); entity.setMpackName("TestMpack2"); entity.setMpackVersion("3.0"); @@ -124,7 +124,7 @@ public void testGetResourcesMpacks() throws Exception { Assert.assertEquals(2, resources.size()); for (Resource resource : resources) { - Long mpackId = (Long) resource.getPropertyValue(MpackResourceProvider.MPACK_ID); + Long mpackId = (Long) resource.getPropertyValue(MpackResourceProvider.MPACK_RESOURCE_ID); if (mpackId == (long) 1) { Assert.assertEquals(resourceExpected1.getPropertyValue(MpackResourceProvider.MPACK_NAME), (String) resource.getPropertyValue(MpackResourceProvider.MPACK_NAME)); Assert.assertEquals(resourceExpected1.getPropertyValue(MpackResourceProvider.MPACK_VERSION), (String) resource.getPropertyValue(MpackResourceProvider.MPACK_VERSION)); @@ -149,11 +149,11 @@ public void testGetResourcesMpackId() throws Exception { Resource.Type type = Resource.Type.Mpack; Predicate predicate = new PredicateBuilder().property( - MpackResourceProvider.MPACK_ID).equals( + MpackResourceProvider.MPACK_RESOURCE_ID).equals( Long.valueOf(1).toString()).toPredicate(); MpackEntity entity = new MpackEntity(); - entity.setMpackId((long) 1); + entity.setId((long) 1); entity.setMpackUri("abcd.tar.gz"); entity.setMpackName("TestMpack1"); entity.setMpackVersion("3.0"); @@ -168,7 +168,7 @@ public void testGetResourcesMpackId() throws Exception { packletArrayList.add(module); Resource resourceExpected1 = new ResourceImpl(Resource.Type.Mpack); - resourceExpected1.setProperty(MpackResourceProvider.MPACK_ID, (long)1); + resourceExpected1.setProperty(MpackResourceProvider.MPACK_RESOURCE_ID, (long)1); resourceExpected1.setProperty(MpackResourceProvider.MPACK_NAME, "TestMpack1"); resourceExpected1.setProperty(MpackResourceProvider.MPACK_VERSION, "3.0"); resourceExpected1.setProperty(MpackResourceProvider.MPACK_URI, "abcd.tar.gz"); @@ -233,7 +233,7 @@ public void testCreateResources() throws Exception { Assert.assertEquals(1,associatedResources.size()); for(Resource r : associatedResources){ - Assert.assertEquals((long)100,r.getPropertyValue(MpackResourceProvider.MPACK_ID)); + Assert.assertEquals((long)100,r.getPropertyValue(MpackResourceProvider.MPACK_RESOURCE_ID)); Assert.assertEquals("testMpack",r.getPropertyValue(MpackResourceProvider.MPACK_NAME)); Assert.assertEquals("3.0",r.getPropertyValue(MpackResourceProvider.MPACK_VERSION)); Assert.assertEquals("../../../../../../../resources/mpacks-v2/abc.tar.gz",r.getPropertyValue(MpackResourceProvider.MPACK_URI)); @@ -250,7 +250,7 @@ public void testCreateResources() throws Exception { public Mpack setupMpack() { Mpack mpack = new Mpack(); - mpack.setMpackId((long)100); + mpack.setResourceId((long)100); mpack.setModules(new ArrayList()); mpack.setPrerequisites(new HashMap()); mpack.setRegistryId(new Long(100)); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java index e464329b4aa..4c7f1441569 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/PreUpgradeCheckResourceProviderTest.java @@ -123,7 +123,7 @@ public void testGetResources() throws Exception{ expect(repoDao.findByPK(1L)).andReturn(repo).anyTimes(); expect(repo.getStackId()).andReturn(targetStackId).atLeastOnce(); - expect(upgradeHelper.suggestUpgradePack("Cluster100", currentStackId, targetStackId, Direction.UPGRADE, UpgradeType.NON_ROLLING, "upgrade_pack11")).andReturn(upgradePack); + expect(upgradeHelper.suggestUpgradePack("Cluster100", currentStackId, targetStackId, Direction.UPGRADE, UpgradeType.EXPRESS, "upgrade_pack11")).andReturn(upgradePack); List upgradeChecksToRun = new LinkedList<>(); List prerequisiteChecks = new LinkedList<>(); @@ -146,7 +146,7 @@ public void testGetResources() throws Exception{ PredicateBuilder builder = new PredicateBuilder(); Predicate predicate = builder.property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CLUSTER_NAME_PROPERTY_ID).equals("Cluster100").and() .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_PACK_PROPERTY_ID).equals("upgrade_pack11").and() - .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_TYPE_PROPERTY_ID).equals(UpgradeType.NON_ROLLING).and() + .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_TYPE_PROPERTY_ID).equals(UpgradeType.EXPRESS).and() .property(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_TARGET_REPOSITORY_VERSION_ID_ID).equals("1").toPredicate(); @@ -174,7 +174,7 @@ public void testGetResources() throws Exception{ String clusterName = (String) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_CLUSTER_NAME_PROPERTY_ID); Assert.assertEquals("Cluster100", clusterName); UpgradeType upgradeType = (UpgradeType) resource.getPropertyValue(PreUpgradeCheckResourceProvider.UPGRADE_CHECK_UPGRADE_TYPE_PROPERTY_ID); - Assert.assertEquals(UpgradeType.NON_ROLLING, upgradeType); + Assert.assertEquals(UpgradeType.EXPRESS, upgradeType); } // verify diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceDependencyResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceDependencyResourceProviderTest.java index 391d9bca60a..f4d4d2d0dbb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceDependencyResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceDependencyResourceProviderTest.java @@ -349,8 +349,8 @@ private Cluster createDefaultCluster(String clusterName) throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion206; - ServiceGroup serviceGroupCore = cluster.addServiceGroup(SERVICE_GROUP_NAME_CORE); - ServiceGroup serviceGroupTest = cluster.addServiceGroup(SERVICE_GROUP_NAME_TEST); + ServiceGroup serviceGroupCore = cluster.addServiceGroup(SERVICE_GROUP_NAME_CORE, "HDP-1.0"); + ServiceGroup serviceGroupTest = cluster.addServiceGroup(SERVICE_GROUP_NAME_TEST, "HDP-1.0"); Service hdfs = cluster.addService(serviceGroupCore, SERVICE_NAME_HDFS, SERVICE_NAME_HDFS, repositoryVersion); Service yarn = cluster.addService(serviceGroupCore, SERVICE_NAME_YARN, SERVICE_NAME_YARN, repositoryVersion); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupDependencyResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupDependencyResourceProviderTest.java index bed36895828..9f1910e9f4f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupDependencyResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupDependencyResourceProviderTest.java @@ -344,8 +344,8 @@ private Cluster createDefaultCluster(String clusterName) throws Exception { RepositoryVersionEntity repositoryVersion = repositoryVersion206; - ServiceGroup serviceGroupCore = cluster.addServiceGroup(SERVICE_GROUP_NAME_CORE); - ServiceGroup serviceGroupTest = cluster.addServiceGroup(SERVICE_GROUP_NAME_TEST); + ServiceGroup serviceGroupCore = cluster.addServiceGroup(SERVICE_GROUP_NAME_CORE, "HDP-2.0.6"); + ServiceGroup serviceGroupTest = cluster.addServiceGroup(SERVICE_GROUP_NAME_TEST, "HDP-2.0.6"); return cluster; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProviderTest.java index d0b2dc4ce2c..d590b4c3c19 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ServiceGroupResourceProviderTest.java @@ -17,13 +17,45 @@ */ package org.apache.ambari.server.controller.internal; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.Set; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.api.resources.ResourceInstanceFactory; +import org.apache.ambari.server.api.resources.ResourceInstanceFactoryImpl; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.api.services.NamedPropertySet; +import org.apache.ambari.server.api.services.RequestBody; +import org.apache.ambari.server.api.services.parsers.JsonRequestBodyParser; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ServiceGroupRequest; +import org.apache.ambari.server.controller.ServiceGroupResponse; +import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.spi.RequestStatus; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.Schema; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.security.TestAuthenticationFactory; import org.apache.ambari.server.security.authorization.AuthorizationException; +import org.apache.ambari.server.security.authorization.AuthorizationHelperInitializer; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ServiceGroup; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; public class ServiceGroupResourceProviderTest { @@ -38,8 +70,76 @@ public static void createServiceGroups(AmbariManagementController controller, Se public static void createServiceGroup(AmbariManagementController controller, String clusterName, String serviceGroupName) throws AmbariException, AuthorizationException { - ServiceGroupRequest request = new ServiceGroupRequest(clusterName, serviceGroupName); + ServiceGroupRequest request = new ServiceGroupRequest(clusterName, serviceGroupName, "dummy-stack-name"); createServiceGroups(controller, Collections.singleton(request)); } + @Before + public void before() { + Authentication authentication = TestAuthenticationFactory.createClusterAdministrator(); + AuthorizationHelperInitializer.viewInstanceDAOReturningNull(); + SecurityContextHolder.getContext().setAuthentication(authentication); + } + + @After + public void clearAuthentication() { + SecurityContextHolder.getContext().setAuthentication(null); + } + + @Test + public void testCreateServiceGroupsWithStackId() throws Exception { + String body = "[{\"ServiceGroupInfo\":{\"service_group_name\": \"CORE\",\"stack_id\": \"HDP-1.2.3\"}}]"; + String clusterName = "c1"; + JsonRequestBodyParser jsonParser = new JsonRequestBodyParser(); + RequestBody requestBody = jsonParser.parse(body).iterator().next(); + //RequestFactory requestFactory = new RequestFactory(); + ResourceInstanceFactory resourceFactory = new ResourceInstanceFactoryImpl(); + Map mapIds = new HashMap<>(); + mapIds.put(Resource.Type.Cluster, clusterName); + mapIds.put(Resource.Type.ServiceGroup, null); + ResourceInstance resource = resourceFactory.createResource(Resource.Type.ServiceGroup, mapIds); + Map mapResourceIds = resource.getKeyValueMap(); + mapResourceIds.put(Resource.Type.Cluster, "c1"); + AmbariManagementController ambariManagementController = createMock(AmbariManagementController.class); + AmbariMetaInfo ambariMetaInfo = createMock(AmbariMetaInfo.class); + Clusters clusters = createMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + Schema serviceGroupSchema = createNiceMock("ServiceGroupSchema", Schema.class); + ClusterController clusterController = createNiceMock(ClusterController.class); + ServiceGroup coreServiceGroup = createNiceMock(ServiceGroup.class); + ServiceGroup edmServiceGroup = createNiceMock(ServiceGroup.class); + ServiceGroupResponse coreServiceGroupResponse = new ServiceGroupResponse(1l, "c1", 1l, "CORE", "HDP-1.2.3"); + expect(ambariManagementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(ambariManagementController.getClusters()).andReturn(clusters).anyTimes(); + expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes(); + expect(cluster.addServiceGroup("CORE", "HDP-1.2.3")).andReturn(coreServiceGroup).anyTimes(); + expect(coreServiceGroup.convertToResponse()).andReturn(coreServiceGroupResponse).anyTimes(); + expect(clusterController.getSchema(Resource.Type.ServiceGroup)).andReturn(serviceGroupSchema).anyTimes(); + expect(serviceGroupSchema.getKeyPropertyId(Resource.Type.Cluster)).andReturn("ServiceGroupInfo/cluster_name").anyTimes(); + expect(serviceGroupSchema.getKeyPropertyId(Resource.Type.ServiceGroup)).andReturn("ServiceGroupInfo/service_group_name").anyTimes(); + expect(serviceGroupSchema.getKeyTypes()).andReturn(Collections.singleton(Resource.Type.ServiceGroup)).anyTimes(); + + replay(cluster, clusters, ambariManagementController, ambariMetaInfo, serviceGroupSchema, clusterController, coreServiceGroup, edmServiceGroup); + + ServiceGroupResourceProvider serviceGroupResourceProvider = new ServiceGroupResourceProvider(ambariManagementController); + Set setProperties = requestBody.getNamedPropertySets(); + if (setProperties.isEmpty()) { + requestBody.addPropertySet(new NamedPropertySet("", new HashMap<>())); + } + for (NamedPropertySet propertySet : setProperties) { + for (Map.Entry entry : mapResourceIds.entrySet()) { + Map mapProperties = propertySet.getProperties(); + String property = serviceGroupSchema.getKeyPropertyId(entry.getKey()); + if (!mapProperties.containsKey(property)) { + mapProperties.put(property, entry.getValue()); + } + } + } + + RequestStatus requestStatus = serviceGroupResourceProvider.createResources(PropertyHelper.getCreateRequest(requestBody.getPropertySets(), requestBody.getRequestInfoProperties())); + Assert.assertEquals(requestStatus.getStatus(), RequestStatus.Status.Complete); + + verify(cluster, clusters, ambariManagementController, ambariMetaInfo, serviceGroupSchema, clusterController, coreServiceGroup, edmServiceGroup); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java index d94810c7ee4..c9464061c9a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/StackDefinedPropertyProviderTest.java @@ -140,7 +140,7 @@ public void setup() throws Exception { cluster.setDesiredStackVersion(stackId); RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service service = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); service.addServiceComponent("NAMENODE"); service.addServiceComponent("DATANODE"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java deleted file mode 100644 index 931b3502871..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ /dev/null @@ -1,2085 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.controller.internal; - -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FileInputStream; -import java.lang.reflect.Field; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.Role; -import org.apache.ambari.server.RoleCommand; -import org.apache.ambari.server.actionmanager.ActionManager; -import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; -import org.apache.ambari.server.actionmanager.ExecutionCommandWrapperFactory; -import org.apache.ambari.server.actionmanager.HostRoleCommand; -import org.apache.ambari.server.actionmanager.HostRoleStatus; -import org.apache.ambari.server.actionmanager.Stage; -import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; -import org.apache.ambari.server.audit.AuditLogger; -import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; -import org.apache.ambari.server.controller.ResourceProviderFactory; -import org.apache.ambari.server.controller.spi.Predicate; -import org.apache.ambari.server.controller.spi.Request; -import org.apache.ambari.server.controller.spi.RequestStatus; -import org.apache.ambari.server.controller.spi.Resource; -import org.apache.ambari.server.controller.spi.Resource.Type; -import org.apache.ambari.server.controller.spi.ResourceProvider; -import org.apache.ambari.server.controller.spi.SystemException; -import org.apache.ambari.server.controller.utilities.PredicateBuilder; -import org.apache.ambari.server.controller.utilities.PropertyHelper; -import org.apache.ambari.server.events.publishers.AmbariEventPublisher; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.dao.ExecutionCommandDAO; -import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; -import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; -import org.apache.ambari.server.orm.dao.RequestDAO; -import org.apache.ambari.server.orm.dao.StackDAO; -import org.apache.ambari.server.orm.dao.StageDAO; -import org.apache.ambari.server.orm.dao.UpgradeDAO; -import org.apache.ambari.server.orm.entities.ExecutionCommandEntity; -import org.apache.ambari.server.orm.entities.HostRoleCommandEntity; -import org.apache.ambari.server.orm.entities.RepoDefinitionEntity; -import org.apache.ambari.server.orm.entities.RepoOsEntity; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.RequestEntity; -import org.apache.ambari.server.orm.entities.StackEntity; -import org.apache.ambari.server.orm.entities.StageEntity; -import org.apache.ambari.server.orm.entities.UpgradeEntity; -import org.apache.ambari.server.orm.entities.UpgradeGroupEntity; -import org.apache.ambari.server.orm.entities.UpgradeHistoryEntity; -import org.apache.ambari.server.orm.entities.UpgradeItemEntity; -import org.apache.ambari.server.security.TestAuthenticationFactory; -import org.apache.ambari.server.serveraction.upgrades.AutoSkipFailedSummaryAction; -import org.apache.ambari.server.serveraction.upgrades.ConfigureAction; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigFactory; -import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.HostState; -import org.apache.ambari.server.state.RepositoryType; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceGroup; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.UpgradeContext; -import org.apache.ambari.server.state.UpgradeHelper; -import org.apache.ambari.server.state.UpgradeState; -import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; -import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; -import org.apache.ambari.server.topology.TopologyManager; -import org.apache.ambari.server.utils.StageUtils; -import org.apache.ambari.server.view.ViewRegistry; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.easymock.EasyMock; -import org.easymock.EasyMockSupport; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.springframework.security.core.context.SecurityContextHolder; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.Lists; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.util.Modules; - -/** - * UpgradeResourceDefinition tests. - */ -public class UpgradeResourceProviderTest extends EasyMockSupport { - - private UpgradeDAO upgradeDao = null; - private RequestDAO requestDao = null; - private RepositoryVersionDAO repoVersionDao = null; - private Injector injector; - private Clusters clusters; - private AmbariManagementController amc; - private ConfigHelper configHelper; - private StackDAO stackDAO; - private TopologyManager topologyManager; - private ConfigFactory configFactory; - private HostRoleCommandDAO hrcDAO; - - RepositoryVersionEntity repoVersionEntity2110; - RepositoryVersionEntity repoVersionEntity2111; - RepositoryVersionEntity repoVersionEntity2112; - RepositoryVersionEntity repoVersionEntity2200; - - /** - * Creates a single host cluster with ZOOKEEPER_SERVER and ZOOKEEPER_CLIENT on - * {@link #repoVersionEntity2110}. - * - * @throws Exception - */ - @Before - public void before() throws Exception { - SecurityContextHolder.getContext().setAuthentication( - TestAuthenticationFactory.createAdministrator()); - - // setup the config helper for placeholder resolution - configHelper = EasyMock.createNiceMock(ConfigHelper.class); - - expect( - configHelper.getPlaceholderValueFromDesiredConfigurations( - EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn( - "placeholder-rendered-properly").anyTimes(); - - expect( - configHelper.getDefaultProperties(EasyMock.anyObject(StackId.class), - EasyMock.anyString())).andReturn( - new HashMap<>()).anyTimes(); - - replay(configHelper); - - InMemoryDefaultTestModule module = new InMemoryDefaultTestModule(); - - // create an injector which will inject the mocks - injector = Guice.createInjector( - Modules.override(module).with(new MockModule())); - - H2DatabaseCleaner.resetSequences(injector); - injector.getInstance(GuiceJpaInitializer.class); - - amc = injector.getInstance(AmbariManagementController.class); - configFactory = injector.getInstance(ConfigFactory.class); - - Field field = AmbariServer.class.getDeclaredField("clusterController"); - field.setAccessible(true); - field.set(null, amc); - - stackDAO = injector.getInstance(StackDAO.class); - upgradeDao = injector.getInstance(UpgradeDAO.class); - requestDao = injector.getInstance(RequestDAO.class); - repoVersionDao = injector.getInstance(RepositoryVersionDAO.class); - hrcDAO = injector.getInstance(HostRoleCommandDAO.class); - - AmbariEventPublisher publisher = EasyMock.createNiceMock(AmbariEventPublisher.class); - replay(publisher); - ViewRegistry.initInstance(new ViewRegistry(publisher)); - - // TODO AMARI-12698, this file is attempting to check RU on version 2.1.1, which doesn't support it - // because it has no upgrade packs. We should use correct versions that have stacks. - // For now, Ignore the tests that fail. - StackEntity stackEntity211 = stackDAO.find("HDP", "2.1.1"); - StackEntity stackEntity220 = stackDAO.find("HDP", "2.2.0"); - StackId stack211 = new StackId(stackEntity211); - - repoVersionEntity2110 = new RepositoryVersionEntity(); - repoVersionEntity2110.setDisplayName("My New Version 1"); - repoVersionEntity2110.addRepoOsEntities(createTestOperatingSystems()); - repoVersionEntity2110.setStack(stackEntity211); - repoVersionEntity2110.setVersion("2.1.1.0"); - repoVersionDao.create(repoVersionEntity2110); - - repoVersionEntity2111 = new RepositoryVersionEntity(); - repoVersionEntity2111.setDisplayName("My New Version 2 for minor upgrade"); - repoVersionEntity2111.addRepoOsEntities(createTestOperatingSystems()); - repoVersionEntity2111.setStack(stackEntity211); - repoVersionEntity2111.setVersion("2.1.1.1"); - repoVersionDao.create(repoVersionEntity2111); - - repoVersionEntity2112 = new RepositoryVersionEntity(); - repoVersionEntity2112.setDisplayName("My New Version 3 for patch upgrade"); - repoVersionEntity2112.addRepoOsEntities(createTestOperatingSystems()); - repoVersionEntity2112.setStack(stackEntity211); - repoVersionEntity2112.setVersion("2.1.1.2"); - repoVersionEntity2112.setType(RepositoryType.PATCH); - repoVersionEntity2112.setVersionXml(""); - repoVersionDao.create(repoVersionEntity2112); - - repoVersionEntity2200 = new RepositoryVersionEntity(); - repoVersionEntity2200.setDisplayName("My New Version 4 for major upgrade"); - repoVersionEntity2200.addRepoOsEntities(createTestOperatingSystems()); - repoVersionEntity2200.setStack(stackEntity220); - repoVersionEntity2200.setVersion("2.2.0.0"); - repoVersionDao.create(repoVersionEntity2200); - - clusters = injector.getInstance(Clusters.class); - - clusters.addCluster("c1", stack211); - Cluster cluster = clusters.getCluster("c1"); - - clusters.addHost("h1"); - Host host = clusters.getHost("h1"); - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6.3"); - host.setHostAttributes(hostAttributes); - host.setState(HostState.HEALTHY); - - clusters.mapHostToCluster("h1", "c1"); - - // add a single ZK server and client on 2.1.1.0 - ServiceGroup serviceGroup = cluster.addServiceGroup(UpgradeResourceProvider.DUMMY_SERVICE_GROUP); - Service service = cluster.addService(serviceGroup, "ZOOKEEPER", "ZOOKEEPER", repoVersionEntity2110); - ServiceComponent component = service.addServiceComponent("ZOOKEEPER_SERVER"); - ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - component = service.addServiceComponent("ZOOKEEPER_CLIENT"); - sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - Configuration configuration = injector.getInstance(Configuration.class); - configuration.setProperty("upgrade.parameter.zk-server.timeout", "824"); - - topologyManager = injector.getInstance(TopologyManager.class); - StageUtils.setTopologyManager(topologyManager); - StageUtils.setConfiguration(configuration); - ActionManager.setTopologyManager(topologyManager); - EasyMock.replay(injector.getInstance(AuditLogger.class)); - } - - private List createTestOperatingSystems() { - List operatingSystems = new ArrayList<>(); - RepoDefinitionEntity repoDefinitionEntity1 = new RepoDefinitionEntity(); - repoDefinitionEntity1.setRepoID("HDP-UTILS"); - repoDefinitionEntity1.setBaseUrl(""); - repoDefinitionEntity1.setRepoName("HDP-UTILS"); - RepoDefinitionEntity repoDefinitionEntity2 = new RepoDefinitionEntity(); - repoDefinitionEntity2.setRepoID("HDP"); - repoDefinitionEntity2.setBaseUrl(""); - repoDefinitionEntity2.setRepoName("HDP"); - RepoOsEntity repoOsEntity = new RepoOsEntity(); - repoOsEntity.setFamily("redhat6"); - repoOsEntity.setAmbariManaged(true); - repoOsEntity.addRepoDefinition(repoDefinitionEntity1); - repoOsEntity.addRepoDefinition(repoDefinitionEntity2); - operatingSystems.add(repoOsEntity); - return operatingSystems; - } - - @After - public void after() throws AmbariException, SQLException { - H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); - EasyMock.reset(injector.getInstance(AuditLogger.class)); - injector = null; - } - - /** - * Obtain request id from the {@code RequestStatus} - * @param requestStatus reqult of the {@code createResources} - * @return id of the request - */ - private long getRequestId(RequestStatus requestStatus){ - assertEquals(1, requestStatus.getAssociatedResources().size()); - Resource r = requestStatus.getAssociatedResources().iterator().next(); - String id = r.getPropertyValue("Upgrade/request_id").toString(); - return Long.parseLong(id); - } - - @Test - public void testCreateResourcesWithAutoSkipFailures() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.ROLLING.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_MANUAL_VERIFICATION, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity entity = upgrades.get(0); - assertTrue(cluster.getClusterId().longValue() == entity.getClusterId().longValue()); - - List upgradeGroups = entity.getUpgradeGroups(); - assertEquals(3, upgradeGroups.size()); - - UpgradeGroupEntity preClusterGroup = upgradeGroups.get(0); - assertEquals("PRE_CLUSTER", preClusterGroup.getName()); - - List preClusterUpgradeItems = preClusterGroup.getItems(); - assertEquals(2, preClusterUpgradeItems.size()); - assertEquals("Foo", parseSingleMessage(preClusterUpgradeItems.get(0).getText())); - assertEquals("Foo", parseSingleMessage(preClusterUpgradeItems.get(1).getText())); - - UpgradeGroupEntity zookeeperGroup = upgradeGroups.get(1); - assertEquals("ZOOKEEPER", zookeeperGroup.getName()); - - List zookeeperUpgradeItems = zookeeperGroup.getItems(); - assertEquals(5, zookeeperUpgradeItems.size()); - - assertEquals("This is a manual task with a placeholder of placeholder-rendered-properly", - parseSingleMessage(zookeeperUpgradeItems.get(0).getText())); - assertEquals("Restarting ZooKeeper Server on h1", zookeeperUpgradeItems.get(1).getText()); - assertEquals("Updating configuration zookeeper-newconfig", - zookeeperUpgradeItems.get(2).getText()); - assertEquals("Service Check ZooKeeper", zookeeperUpgradeItems.get(3).getText()); - assertTrue(zookeeperUpgradeItems.get(4).getText().contains("There are failures that were automatically skipped")); - - // the last upgrade item is the skipped failure check - UpgradeItemEntity skippedFailureCheck = zookeeperUpgradeItems.get(zookeeperUpgradeItems.size() - 1); - skippedFailureCheck.getTasks().contains(AutoSkipFailedSummaryAction.class.getName()); - - UpgradeGroupEntity postClusterGroup = upgradeGroups.get(2); - assertEquals("POST_CLUSTER", postClusterGroup.getName()); - - List postClusterUpgradeItems = postClusterGroup.getItems(); - assertEquals(2, postClusterUpgradeItems.size()); - assertEquals("Please confirm you are ready to finalize", parseSingleMessage(postClusterUpgradeItems.get(0).getText())); - assertEquals("Save Cluster State", postClusterUpgradeItems.get(1).getText()); - } - - @Test - public void testCreateResourcesWithAutoSkipManualVerification() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.ROLLING.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_MANUAL_VERIFICATION, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity entity = upgrades.get(0); - assertTrue(cluster.getClusterId().longValue() == entity.getClusterId().longValue()); - - List upgradeGroups = entity.getUpgradeGroups(); - assertEquals(2, upgradeGroups.size()); - - UpgradeGroupEntity zookeeperGroup = upgradeGroups.get(0); - assertEquals("ZOOKEEPER", zookeeperGroup.getName()); - - List zookeeperUpgradeItems = zookeeperGroup.getItems(); - assertEquals(3, zookeeperUpgradeItems.size()); - assertEquals("Restarting ZooKeeper Server on h1", zookeeperUpgradeItems.get(0).getText()); - assertEquals("Updating configuration zookeeper-newconfig", - zookeeperUpgradeItems.get(1).getText()); - assertEquals("Service Check ZooKeeper", zookeeperUpgradeItems.get(2).getText()); - - UpgradeGroupEntity postClusterGroup = upgradeGroups.get(1); - assertEquals("POST_CLUSTER", postClusterGroup.getName()); - - List postClusterUpgradeItems = postClusterGroup.getItems(); - assertEquals(1, postClusterUpgradeItems.size()); - assertEquals("Save Cluster State", postClusterUpgradeItems.get(0).getText()); - } - - @Test - public void testCreateResourcesWithAutoSkipAll() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.ROLLING.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_MANUAL_VERIFICATION, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity entity = upgrades.get(0); - assertTrue(cluster.getClusterId().longValue() == entity.getClusterId().longValue()); - - List upgradeGroups = entity.getUpgradeGroups(); - assertEquals(2, upgradeGroups.size()); - - UpgradeGroupEntity zookeeperGroup = upgradeGroups.get(0); - assertEquals("ZOOKEEPER", zookeeperGroup.getName()); - - List zookeeperUpgradeItems = zookeeperGroup.getItems(); - assertEquals(4, zookeeperUpgradeItems.size()); - - assertEquals("Restarting ZooKeeper Server on h1", zookeeperUpgradeItems.get(0).getText()); - assertEquals("Updating configuration zookeeper-newconfig", - zookeeperUpgradeItems.get(1).getText()); - assertEquals("Service Check ZooKeeper", zookeeperUpgradeItems.get(2).getText()); - assertTrue(zookeeperUpgradeItems.get(3).getText().contains("There are failures that were automatically skipped")); - - // the last upgrade item is the skipped failure check - UpgradeItemEntity skippedFailureCheck = zookeeperUpgradeItems.get(zookeeperUpgradeItems.size() - 1); - skippedFailureCheck.getTasks().contains(AutoSkipFailedSummaryAction.class.getName()); - - UpgradeGroupEntity postClusterGroup = upgradeGroups.get(1); - assertEquals("POST_CLUSTER", postClusterGroup.getName()); - - List postClusterUpgradeItems = postClusterGroup.getItems(); - assertEquals(1, postClusterUpgradeItems.size()); - assertEquals("Save Cluster State", postClusterUpgradeItems.get(0).getText()); - } - - @Test - public void testGetResources() throws Exception { - RequestStatus status = testCreateResources(); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - // upgrade - Set propertyIds = new HashSet<>(); - propertyIds.add("Upgrade"); - - Predicate predicate = new PredicateBuilder() - .property(UpgradeResourceProvider.UPGRADE_REQUEST_ID).equals("1").and() - .property(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - Request request = PropertyHelper.getReadRequest(propertyIds); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Set resources = upgradeResourceProvider.getResources(request, predicate); - - assertEquals(1, resources.size()); - res = resources.iterator().next(); - assertNotNull(res.getPropertyValue("Upgrade/progress_percent")); - assertNotNull(res.getPropertyValue(UpgradeResourceProvider.UPGRADE_DIRECTION)); - assertEquals(Direction.UPGRADE, res.getPropertyValue(UpgradeResourceProvider.UPGRADE_DIRECTION)); - assertEquals(false, res.getPropertyValue(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES)); - assertEquals(false, res.getPropertyValue(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES)); - assertEquals(UpgradeType.ROLLING, res.getPropertyValue(UpgradeResourceProvider.UPGRADE_TYPE)); - - // upgrade groups - propertyIds.clear(); - propertyIds.add("UpgradeGroup"); - - predicate = new PredicateBuilder() - .property(UpgradeGroupResourceProvider.UPGRADE_REQUEST_ID).equals("1").and() - .property(UpgradeGroupResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - request = PropertyHelper.getReadRequest(propertyIds); - - ResourceProvider upgradeGroupResourceProvider = new UpgradeGroupResourceProvider(amc); - resources = upgradeGroupResourceProvider.getResources(request, predicate); - - assertEquals(3, resources.size()); - res = resources.iterator().next(); - assertNotNull(res.getPropertyValue("UpgradeGroup/status")); - assertNotNull(res.getPropertyValue("UpgradeGroup/group_id")); - assertNotNull(res.getPropertyValue("UpgradeGroup/total_task_count")); - assertNotNull(res.getPropertyValue("UpgradeGroup/in_progress_task_count")); - assertNotNull(res.getPropertyValue("UpgradeGroup/completed_task_count")); - - // upgrade items - propertyIds.clear(); - propertyIds.add("UpgradeItem"); - - predicate = new PredicateBuilder() - .property(UpgradeItemResourceProvider.UPGRADE_GROUP_ID).equals("1").and() - .property(UpgradeItemResourceProvider.UPGRADE_REQUEST_ID).equals("1").and() - .property(UpgradeItemResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - request = PropertyHelper.getReadRequest(propertyIds); - - ResourceProvider upgradeItemResourceProvider = new UpgradeItemResourceProvider(amc); - resources = upgradeItemResourceProvider.getResources(request, predicate); - - assertEquals(2, resources.size()); - - res = resources.iterator().next(); - assertNotNull(res.getPropertyValue("UpgradeItem/status")); - - // !!! check for manual stage vs item text - propertyIds.clear(); - propertyIds.add("UpgradeItem"); - - predicate = new PredicateBuilder() - .property(UpgradeItemResourceProvider.UPGRADE_GROUP_ID).equals("3").and() - .property(UpgradeItemResourceProvider.UPGRADE_REQUEST_ID).equals("1").and() - .property(UpgradeItemResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - request = PropertyHelper.getReadRequest(propertyIds); - - upgradeItemResourceProvider = new UpgradeItemResourceProvider(amc); - resources = upgradeItemResourceProvider.getResources(request, predicate); - assertEquals(2, resources.size()); - res = resources.iterator().next(); - - assertEquals("Confirm Finalize", res.getPropertyValue("UpgradeItem/context")); - String msgStr = res.getPropertyValue("UpgradeItem/text").toString(); - JsonParser parser = new JsonParser(); - JsonArray msgArray = (JsonArray) parser.parse(msgStr); - JsonObject msg = (JsonObject) msgArray.get(0); - - assertTrue(msg.get("message").getAsString().startsWith("Please confirm")); - } - - /** - * Tests that retrieving an upgrade correctly populates less common upgrade - * options correctly. - */ - @Test - public void testGetResourcesWithSpecialOptions() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2111.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - // tests skipping SC failure options - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, "true"); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - RequestStatus status = upgradeResourceProvider.createResources(request); - assertNotNull(status); - - // upgrade - Set propertyIds = new HashSet<>(); - propertyIds.add("Upgrade"); - - Predicate predicate = new PredicateBuilder() - .property(UpgradeResourceProvider.UPGRADE_REQUEST_ID).equals("1").and() - .property(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - - request = PropertyHelper.getReadRequest(propertyIds); - Set resources = upgradeResourceProvider.getResources(request, predicate); - - assertEquals(1, resources.size()); - Resource resource = resources.iterator().next(); - - assertEquals(true, resource.getPropertyValue(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES)); - assertEquals(true, resource.getPropertyValue(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES)); - } - - - @Test - public void testCreatePartialDowngrade() throws Exception { - clusters.addHost("h2"); - Host host = clusters.getHost("h2"); - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6.3"); - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster("h2", "c1"); - Cluster cluster = clusters.getCluster("c1"); - Service service = cluster.getService("ZOOKEEPER"); - - // start out with 0 (sanity check) - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - // a downgrade MUST have an upgrade to come from, so populate an upgrade in - // the DB - RequestEntity requestEntity = new RequestEntity(); - requestEntity.setRequestId(2L); - requestEntity.setClusterId(cluster.getClusterId()); - requestEntity.setStages(new ArrayList<>()); - requestDao.create(requestEntity); - - UpgradeEntity upgradeEntity = new UpgradeEntity(); - upgradeEntity.setClusterId(cluster.getClusterId()); - upgradeEntity.setDirection(Direction.UPGRADE); - upgradeEntity.setRepositoryVersion(repoVersionEntity2200); - upgradeEntity.setUpgradePackage("upgrade_test"); - upgradeEntity.setUpgradeType(UpgradeType.ROLLING); - upgradeEntity.setRequestEntity(requestEntity); - - UpgradeHistoryEntity history = new UpgradeHistoryEntity(); - history.setUpgrade(upgradeEntity); - history.setFromRepositoryVersion(service.getDesiredRepositoryVersion()); - history.setTargetRepositoryVersion(repoVersionEntity2200); - history.setServiceName(service.getName()); - history.setComponentName("ZOKKEEPER_SERVER"); - upgradeEntity.addHistory(history); - - history = new UpgradeHistoryEntity(); - history.setUpgrade(upgradeEntity); - history.setFromRepositoryVersion(service.getDesiredRepositoryVersion()); - history.setTargetRepositoryVersion(repoVersionEntity2200); - history.setServiceName(service.getName()); - history.setComponentName("ZOKKEEPER_CLIENT"); - upgradeEntity.addHistory(history); - - upgradeDao.create(upgradeEntity); - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - // push a ZK server foward to the new repo version, leaving the old one on - // the old version - ServiceComponent component = service.getServiceComponent("ZOOKEEPER_SERVER"); - ServiceComponentHost sch = component.addServiceComponentHost("h2"); - sch.setVersion(repoVersionEntity2200.getVersion()); - - UpgradeEntity lastUpgrade = upgradeDao.findLastUpgradeForCluster(cluster.getClusterId(), Direction.UPGRADE); - assertNotNull(lastUpgrade); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - Map requestInfoProperties = new HashMap<>(); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); - upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(2, upgrades.size()); - - UpgradeEntity downgrade = upgrades.get(1); - assertTrue(cluster.getClusterId().longValue() == downgrade.getClusterId().longValue()); - - List upgradeGroups = downgrade.getUpgradeGroups(); - assertEquals(3, upgradeGroups.size()); - - // the ZK restart group should only have 3 entries since the ZK server on h1 - // didn't get upgraded - UpgradeGroupEntity group = upgradeGroups.get(1); - assertEquals("ZOOKEEPER", group.getName()); - assertEquals(3, group.getItems().size()); - } - - - @Test - public void testDowngradeToBase() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2111.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgrade = upgrades.get(0); - - // now abort the upgrade so another can be created - abortUpgrade(upgrade.getRequestId()); - - // create another upgrade which should fail - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, "9999"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - try { - upgradeResourceProvider.createResources(request); - Assert.fail("Expected an exception going downgrade with no upgrade pack"); - } catch (Exception e) { - // !!! expected - } - - // fix the properties and try again - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - Map requestInfoProperties = new HashMap<>(); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); - RequestStatus status = upgradeResourceProvider.createResources(request); - assertEquals(1, status.getAssociatedResources().size()); - Resource r = status.getAssociatedResources().iterator().next(); - String id = r.getPropertyValue("Upgrade/request_id").toString(); - - UpgradeEntity entity = upgradeDao.findUpgrade(Long.parseLong(id)); - assertNotNull(entity); - assertEquals(Direction.DOWNGRADE, entity.getDirection()); - - // associated version is the FROM on DOWNGRADE - assertEquals(repoVersionEntity2111.getVersion(), entity.getRepositoryVersion().getVersion()); - - // target is by service - assertEquals(repoVersionEntity2110.getVersion(), - entity.getHistory().iterator().next().getTargetVersion()); - - StageDAO dao = injector.getInstance(StageDAO.class); - List stages = dao.findByRequestId(entity.getRequestId()); - - Gson gson = new Gson(); - for (StageEntity se : stages) { - Map map = gson.>fromJson(se.getCommandParamsStage(), Map.class); - assertTrue(map.containsKey("upgrade_direction")); - assertEquals("downgrade", map.get("upgrade_direction")); - } - - } - - - - /** - * Test Downgrade from the partially completed upgrade - */ - @Test - public void testNotFullDowngrade() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - // add additional service for the test - ServiceGroup serviceGroup = cluster.getServiceGroup(UpgradeResourceProvider.DUMMY_SERVICE_GROUP); - Service service = cluster.addService(serviceGroup, "HIVE", "HIVE", repoVersionEntity2110); - - ServiceComponent component = service.addServiceComponent("HIVE_SERVER"); - ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - // create upgrade request - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_nonrolling_new_stack"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, "NON_ROLLING"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - // check that upgrade was created and groups for the tested services are on place - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgrade = upgrades.get(0); - List groups = upgrade.getUpgradeGroups(); - boolean isHiveGroupFound = false; - boolean isZKGroupFound = false; - - // look only for testing groups - for (UpgradeGroupEntity group: groups) { - if (group.getName().equalsIgnoreCase("hive")) { - isHiveGroupFound = true; - } else if (group.getName().equalsIgnoreCase("zookeeper")){ - isZKGroupFound = true; - } - } - - assertTrue(isHiveGroupFound); - assertTrue(isZKGroupFound); - - isHiveGroupFound = false; - isZKGroupFound = false; - sch.setVersion("2.2.0.0"); - - // now abort the upgrade so another can be created - abortUpgrade(upgrade.getRequestId()); - - // create downgrade with one upgraded service - service.setDesiredRepositoryVersion(repoVersionEntity2200); - - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_nonrolling_new_stack"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - Map requestInfoProperties = new HashMap<>(); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); - RequestStatus status = upgradeResourceProvider.createResources(request); - UpgradeEntity upgradeEntity = upgradeDao.findUpgradeByRequestId(getRequestId(status)); - - for (UpgradeGroupEntity group: upgradeEntity.getUpgradeGroups()) { - if (group.getName().equalsIgnoreCase("hive")) { - isHiveGroupFound = true; - } else if (group.getName().equalsIgnoreCase("zookeeper")){ - isZKGroupFound = true; - } - } - - // as services not updated, nothing to downgrade - assertTrue(isHiveGroupFound); - assertFalse(isZKGroupFound); - } - - - @Test - public void testAbort() throws Exception { - RequestStatus status = testCreateResources(); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "true"); - - UpgradeResourceProvider urp = createProvider(amc); - - Request req = PropertyHelper.getUpdateRequest(requestProps, null); - urp.updateResources(req, null); - - List commands = hrcDAO.findByRequest(id); - - int i = 0; - for (HostRoleCommandEntity command : commands) { - if (i < 3) { - command.setStatus(HostRoleStatus.COMPLETED); - } else { - command.setStatus(HostRoleStatus.ABORTED); - } - hrcDAO.merge(command); - i++; - } - - req = PropertyHelper.getReadRequest( - UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, - UpgradeResourceProvider.UPGRADE_ID, - UpgradeResourceProvider.REQUEST_PROGRESS_PERCENT_ID); - - Predicate pred = new PredicateBuilder() - .property(UpgradeResourceProvider.UPGRADE_REQUEST_ID).equals(id.toString()) - .and() - .property(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME).equals("c1") - .toPredicate(); - - Set resources = urp.getResources(req, pred); - assertEquals(1, resources.size()); - res = resources.iterator().next(); - - Double value = (Double) res.getPropertyValue(UpgradeResourceProvider.REQUEST_PROGRESS_PERCENT_ID); - - assertEquals(37.5d, value, 0.1d); - } - - - @Test - public void testRetry() throws Exception { - RequestStatus status = testCreateResources(); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "true"); - - UpgradeResourceProvider urp = createProvider(amc); - - // !!! make sure we can. actual abort is tested elsewhere - Request req = PropertyHelper.getUpdateRequest(requestProps, null); - urp.updateResources(req, null); - - ActionManager am = injector.getInstance(ActionManager.class); - - List commands = am.getRequestTasks(id); - - boolean foundOne = false; - for (HostRoleCommand hrc : commands) { - if (hrc.getRole().equals(Role.AMBARI_SERVER_ACTION)) { - assertEquals(-1L, hrc.getHostId()); - assertNull(hrc.getHostName()); - foundOne = true; - } - } - assertTrue("Expected at least one server-side action", foundOne); - - HostRoleCommand cmd = commands.get(commands.size()-1); - - HostRoleCommandDAO dao = injector.getInstance(HostRoleCommandDAO.class); - HostRoleCommandEntity entity = dao.findByPK(cmd.getTaskId()); - entity.setStatus(HostRoleStatus.ABORTED); - dao.merge(entity); - - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "PENDING"); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "false"); - - // !!! make sure we can. actual reset is tested elsewhere - req = PropertyHelper.getUpdateRequest(requestProps, null); - urp.updateResources(req, null); - } - - @Test(expected = IllegalArgumentException.class) - public void testAbortWithoutSuspendFlag() throws Exception { - RequestStatus status = testCreateResources(); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, id.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED"); - - UpgradeResourceProvider urp = createProvider(amc); - Request req = PropertyHelper.getUpdateRequest(requestProps, null); - urp.updateResources(req, null); - } - - @Test - public void testDirectionUpgrade() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - StackEntity stackEntity = stackDAO.find("HDP", "2.1.1"); - - RepositoryVersionEntity repoVersionEntity = new RepositoryVersionEntity(); - repoVersionEntity.setDisplayName("My New Version 3"); - repoVersionEntity.addRepoOsEntities(new ArrayList<>()); - repoVersionEntity.setStack(stackEntity); - repoVersionEntity.setVersion("2.2.2.3"); - repoVersionDao.create(repoVersionEntity); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_direction"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgrade = upgrades.get(0); - Long id = upgrade.getRequestId(); - assertEquals(3, upgrade.getUpgradeGroups().size()); - // Ensure that there are no items related to downgrade in the upgrade direction - UpgradeGroupEntity group = upgrade.getUpgradeGroups().get(2); - Assert.assertEquals("POST_CLUSTER", group.getName()); - Assert.assertTrue(!group.getItems().isEmpty()); - for (UpgradeItemEntity item : group.getItems()) { - Assert.assertFalse(item.getText().toLowerCase().contains("downgrade")); - } - - // now abort the upgrade so another can be created - abortUpgrade(upgrade.getRequestId()); - - requestProps.clear(); - // Now perform a downgrade - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_direction"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(2, upgrades.size()); - - upgrade = null; - for (UpgradeEntity u : upgrades) { - if (!u.getRequestId().equals(id)) { - upgrade = u; - } - } - assertNotNull(upgrade); - List groups = upgrade.getUpgradeGroups(); - assertEquals("Downgrade groups reduced from 3 to 2", 1, groups.size()); - group = upgrade.getUpgradeGroups().get(0); - assertEquals("Execution items increased from 1 to 2", 2, group.getItems().size()); - } - - - - @Test - public void testPercents() throws Exception { - RequestStatus status = testCreateResources(); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - StageDAO stageDao = injector.getInstance(StageDAO.class); - HostRoleCommandDAO hrcDao = injector.getInstance(HostRoleCommandDAO.class); - - List stages = stageDao.findByRequestId(id); - List tasks = hrcDao.findByRequest(id); - - Set stageIds = new HashSet<>(); - for (StageEntity se : stages) { - stageIds.add(se.getStageId()); - } - - CalculatedStatus calc = null; - int i = 0; - for (HostRoleCommandEntity hrce : tasks) { - hrce.setStatus(HostRoleStatus.IN_PROGRESS); - hrcDao.merge(hrce); - calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds); - assertEquals(((i++) + 1) * 4.375d, calc.getPercent(), 0.01d); - assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus()); - } - - i = 0; - for (HostRoleCommandEntity hrce : tasks) { - hrce.setStatus(HostRoleStatus.COMPLETED); - hrcDao.merge(hrce); - calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds); - assertEquals(35 + (((i++) + 1) * 8.125), calc.getPercent(), 0.01d); - if (i < 8) { - assertEquals(HostRoleStatus.IN_PROGRESS, calc.getStatus()); - } - } - - calc = CalculatedStatus.statusFromStageSummary(hrcDao.findAggregateCounts(id), stageIds); - assertEquals(HostRoleStatus.COMPLETED, calc.getStatus()); - assertEquals(100d, calc.getPercent(), 0.01d); - } - - - @Test - public void testCreateCrossStackUpgrade() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - StackId oldStack = repoVersionEntity2110.getStackId(); - - for (Service s : cluster.getServices().values()) { - assertEquals(oldStack, s.getDesiredStackId()); - - for (ServiceComponent sc : s.getServiceComponents().values()) { - assertEquals(oldStack, sc.getDesiredStackId()); - - for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) { - assertEquals(repoVersionEntity2110.getVersion(), sch.getVersion()); - } - } - } - - - Config config = configFactory.createNew(cluster, "zoo.cfg", "abcdefg", Collections.singletonMap("a", "b"), null); - cluster.addDesiredConfig("admin", Collections.singleton(config)); - - // create the upgrade across major versions - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgrade = upgrades.get(0); - assertEquals(3, upgrade.getUpgradeGroups().size()); - - UpgradeGroupEntity group = upgrade.getUpgradeGroups().get(2); - assertEquals(2, group.getItems().size()); - - group = upgrade.getUpgradeGroups().get(0); - assertEquals(2, group.getItems().size()); - - assertTrue(cluster.getDesiredConfigs().containsKey("zoo.cfg")); - - for (Service s : cluster.getServices().values()) { - assertEquals(repoVersionEntity2200, s.getDesiredRepositoryVersion()); - - for (ServiceComponent sc : s.getServiceComponents().values()) { - assertEquals(repoVersionEntity2200, sc.getDesiredRepositoryVersion()); - } - } - } - - /** - * @param amc - * @return the provider - */ - private UpgradeResourceProvider createProvider(AmbariManagementController amc) { - ResourceProviderFactory factory = injector.getInstance(ResourceProviderFactory.class); - AbstractControllerResourceProvider.init(factory); - - Resource.Type type = Type.Upgrade; - return (UpgradeResourceProvider) AbstractControllerResourceProvider.getResourceProvider(type, amc); - } - - private RequestStatus testCreateResources() throws Exception { - - Cluster cluster = clusters.getCluster("c1"); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2111.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - RequestStatus status = upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity entity = upgrades.get(0); - assertTrue(cluster.getClusterId().longValue() == entity.getClusterId().longValue()); - assertEquals(UpgradeType.ROLLING, entity.getUpgradeType()); - - StageDAO stageDAO = injector.getInstance(StageDAO.class); - List stageEntities = stageDAO.findByRequestId(entity.getRequestId()); - Gson gson = new Gson(); - for (StageEntity se : stageEntities) { - Map map = gson.> fromJson(se.getCommandParamsStage(),Map.class); - assertTrue(map.containsKey("upgrade_direction")); - assertEquals("upgrade", map.get("upgrade_direction")); - - if(map.containsKey("upgrade_type")){ - assertEquals("rolling_upgrade", map.get("upgrade_type")); - } - } - - List upgradeGroups = entity.getUpgradeGroups(); - assertEquals(3, upgradeGroups.size()); - - UpgradeGroupEntity group = upgradeGroups.get(1); - assertEquals(4, group.getItems().size()); - - assertTrue( - group.getItems().get(0).getText().contains("placeholder of placeholder-rendered-properly")); - - assertTrue(group.getItems().get(1).getText().contains("Restarting")); - assertTrue(group.getItems().get(2).getText().contains("Updating")); - assertTrue(group.getItems().get(3).getText().contains("Service Check")); - - ActionManager am = injector.getInstance(ActionManager.class); - List requests = am.getRequestsByStatus( - org.apache.ambari.server.actionmanager.RequestStatus.IN_PROGRESS, 100, true); - - assertEquals(1, requests.size()); - assertEquals(requests.get(0), entity.getRequestId()); - - List stages = am.getRequestStatus(requests.get(0).longValue()); - - assertEquals(8, stages.size()); - - List tasks = am.getRequestTasks(requests.get(0).longValue()); - // same number of tasks as stages here - assertEquals(8, tasks.size()); - - Set slaveStageIds = new HashSet<>(); - - UpgradeGroupEntity coreSlavesGroup = upgradeGroups.get(1); - - for (UpgradeItemEntity itemEntity : coreSlavesGroup.getItems()) { - slaveStageIds.add(itemEntity.getStageId()); - } - - for (Stage stage : stages) { - - // For this test the core slaves group stages should be skippable and NOT - // allow retry. - assertEquals(slaveStageIds.contains(stage.getStageId()), stage.isSkippable()); - - for (Map taskMap : stage.getHostRoleCommands().values()) { - - for (HostRoleCommand task : taskMap.values()) { - assertEquals(!slaveStageIds.contains(stage.getStageId()), task.isRetryAllowed()); - } - } - } - return status; - } - - @Test - public void testUpdateSkipFailures() throws Exception { - testCreateResourcesWithAutoSkipFailures(); - - List upgrades = upgradeDao.findUpgrades(1); - assertEquals(1, upgrades.size()); - - UpgradeEntity entity = upgrades.get(0); - - HostRoleCommandDAO dao = injector.getInstance(HostRoleCommandDAO.class); - - List tasks = dao.findByRequest(entity.getRequestId()); - for (HostRoleCommandEntity task : tasks) { - StageEntity stage = task.getStage(); - if (stage.isSkippable() && stage.isAutoSkipOnFailureSupported()) { - assertTrue(task.isFailureAutoSkipped()); - } else { - assertFalse(task.isFailureAutoSkipped()); - } - } - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, "" + entity.getRequestId()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getUpdateRequest(requestProps, null); - upgradeResourceProvider.updateResources(request, null); - - tasks = dao.findByRequest(entity.getRequestId()); - for (HostRoleCommandEntity task : tasks) { - if (task.getRoleCommand() == RoleCommand.SERVICE_CHECK) { - assertFalse(task.isFailureAutoSkipped()); - } else { - StageEntity stage = task.getStage(); - if (stage.isSkippable() && stage.isAutoSkipOnFailureSupported()) { - assertTrue(task.isFailureAutoSkipped()); - } else { - assertFalse(task.isFailureAutoSkipped()); - } - } - } - - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, "" + entity.getRequestId()); - - request = PropertyHelper.getUpdateRequest(requestProps, null); - upgradeResourceProvider.updateResources(request, null); - - tasks = dao.findByRequest(entity.getRequestId()); - for (HostRoleCommandEntity task : tasks) { - if (task.getRoleCommand() == RoleCommand.SERVICE_CHECK) { - StageEntity stage = task.getStage(); - if (stage.isSkippable() && stage.isAutoSkipOnFailureSupported()) { - assertTrue(task.isFailureAutoSkipped()); - } else { - assertFalse(task.isFailureAutoSkipped()); - } - } else { - assertFalse(task.isFailureAutoSkipped()); - } - } - - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_FAILURES, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_SC_FAILURES, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, "" + entity.getRequestId()); - - request = PropertyHelper.getUpdateRequest(requestProps, null); - upgradeResourceProvider.updateResources(request, null); - - tasks = dao.findByRequest(entity.getRequestId()); - for (HostRoleCommandEntity task : tasks) { - assertFalse(task.isFailureAutoSkipped()); - } - } - - /** - * Tests that an error while commiting the data cleanly rolls back the transaction so that - * no request/stage/tasks are created. - * - * @throws Exception - */ - @Test - public void testRollback() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.ROLLING.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_MANUAL_VERIFICATION, Boolean.FALSE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - // this will cause a NPE when creating the upgrade, allowing us to test - // rollback - UpgradeResourceProvider upgradeResourceProvider = createProvider(amc); - UpgradeResourceProvider.s_upgradeDAO = null; - - try { - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - Assert.fail("Expected a NullPointerException"); - } catch (NullPointerException npe) { - // expected - } - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - List requestIds = requestDao.findAllRequestIds(1, true, cluster.getClusterId()); - assertEquals(0, requestIds.size()); - } - - /** - * Tests that a {@link UpgradeType#HOST_ORDERED} upgrade throws an exception - * on missing hosts. - * - * @throws Exception - */ - @Test() - public void testCreateHostOrderedUpgradeThrowsExceptions() throws Exception { - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_host_ordered"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.HOST_ORDERED.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - - try { - upgradeResourceProvider.createResources(request); - Assert.fail("The request should have failed due to the missing Upgrade/host_order property"); - } catch( SystemException systemException ){ - // expected - } - - // stick a bad host_ordered_hosts in there which has the wrong hosts - Set>> hostsOrder = new LinkedHashSet<>(); - Map> hostGrouping = new HashMap<>(); - hostGrouping.put("hosts", Lists.newArrayList("invalid-host")); - hostsOrder.add(hostGrouping); - - requestProps.put(UpgradeResourceProvider.UPGRADE_HOST_ORDERED_HOSTS, hostsOrder); - - try { - upgradeResourceProvider.createResources(request); - Assert.fail("The request should have failed due to invalid hosts"); - } catch (SystemException systemException) { - // expected - } - - // use correct hosts now - hostsOrder = new LinkedHashSet<>(); - hostGrouping = new HashMap<>(); - hostGrouping.put("hosts", Lists.newArrayList("h1")); - hostsOrder.add(hostGrouping); - - requestProps.put(UpgradeResourceProvider.UPGRADE_HOST_ORDERED_HOSTS, hostsOrder); - upgradeResourceProvider.createResources(request); - - - // make sure that the desired versions are updated - Cluster cluster = clusters.getCluster("c1"); - assertNotNull(cluster); - - Service service = cluster.getService("ZOOKEEPER"); - assertEquals(repoVersionEntity2200, service.getDesiredRepositoryVersion()); - } - - /** - * Exercises that a component that goes from upgrade->downgrade that switches - * {@code versionAdvertised} between will go to UKNOWN. This exercises - * {@link UpgradeHelper#updateDesiredRepositoriesAndConfigs(UpgradeContext)} - * - * @throws Exception - */ - @Test - public void testCreateUpgradeDowngradeCycleAdvertisingVersion() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - ServiceGroup serviceGroup = cluster.getServiceGroup(UpgradeResourceProvider.DUMMY_SERVICE_GROUP); - Service service = cluster.addService(serviceGroup, "STORM", "STORM", repoVersionEntity2110); - - ServiceComponent component = service.addServiceComponent("DRPC_SERVER"); - ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - Map requestInfoProperties = new HashMap<>(); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); - - RequestStatus status = upgradeResourceProvider.createResources(request); - assertEquals(1, status.getAssociatedResources().size()); - - Resource r = status.getAssociatedResources().iterator().next(); - String id = r.getPropertyValue("Upgrade/request_id").toString(); - - component = service.getServiceComponent("DRPC_SERVER"); - assertNotNull(component); - assertEquals("2.2.0.0", component.getDesiredVersion()); - - ServiceComponentHost hostComponent = component.getServiceComponentHost("h1"); - assertEquals(UpgradeState.IN_PROGRESS, hostComponent.getUpgradeState()); - - // !!! can't start a downgrade until cancelling the previous upgrade - abortUpgrade(Long.parseLong(id)); - - requestProps.clear(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), requestInfoProperties); - status = upgradeResourceProvider.createResources(request); - - component = service.getServiceComponent("DRPC_SERVER"); - assertNotNull(component); - assertEquals(repoVersionEntity2110, component.getDesiredRepositoryVersion()); - - hostComponent = component.getServiceComponentHost("h1"); - assertEquals(UpgradeState.NONE, hostComponent.getUpgradeState()); - assertEquals("UNKNOWN", hostComponent.getVersion()); - } - - /** - * Ensures that stages created with an HOU are sequential and do not skip any - * IDs. When there are stages with IDs like (1,2,3,5,6,7,10), the request will - * get stuck in a PENDING state. This affects HOU specifically since they can - * potentially try to create empty stages which won't get persisted (such as a - * STOP on client-only hosts). - * - * @throws Exception - */ - @Test() - public void testEmptyGroupingsDoNotSkipStageIds() throws Exception { - - StageDAO stageDao = injector.getInstance(StageDAO.class); - Assert.assertEquals(0, stageDao.findAll().size()); - - // strip out all non-client components - clients don't have STOP commands - Cluster cluster = clusters.getCluster("c1"); - List schs = cluster.getServiceComponentHosts("h1"); - for (ServiceComponentHost sch : schs) { - if (sch.isClientComponent()) { - continue; - } - - cluster.removeServiceComponentHost(sch); - } - - // define host order - Set>> hostsOrder = new LinkedHashSet<>(); - Map> hostGrouping = new HashMap<>(); - hostGrouping = new HashMap<>(); - hostGrouping.put("hosts", Lists.newArrayList("h1")); - hostsOrder.add(hostGrouping); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_host_ordered"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.HOST_ORDERED.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS,Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - requestProps.put(UpgradeResourceProvider.UPGRADE_HOST_ORDERED_HOSTS, hostsOrder); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List stages = stageDao.findByRequestId(cluster.getUpgradeInProgress().getRequestId()); - Assert.assertEquals(3, stages.size()); - - long expectedStageId = 1L; - for (StageEntity stage : stages) { - Assert.assertEquals(expectedStageId++, stage.getStageId().longValue()); - } - } - - /** - * Tests that from/to repository version history is created correctly on the - * upgrade. - * - * @throws Exception - */ - @Test - public void testUpgradeHistory() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.ROLLING.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_MANUAL_VERIFICATION, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgrade = cluster.getUpgradeInProgress(); - List histories = upgrade.getHistory(); - assertEquals(2, histories.size()); - - for( UpgradeHistoryEntity history : histories){ - assertEquals( "ZOOKEEPER", history.getServiceName() ); - assertEquals(repoVersionEntity2110, history.getFromReposistoryVersion()); - assertEquals(repoVersionEntity2200, history.getTargetRepositoryVersion()); - } - - // abort the upgrade and create the downgrade - abortUpgrade(upgrade.getRequestId()); - - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_nonrolling_new_stack"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); - - Map requestInfoProperties = new HashMap<>(); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), - requestInfoProperties); - RequestStatus status = upgradeResourceProvider.createResources(request); - UpgradeEntity downgrade = upgradeDao.findUpgradeByRequestId(getRequestId(status)); - assertEquals(Direction.DOWNGRADE, downgrade.getDirection()); - - // check from/to history - histories = downgrade.getHistory(); - assertEquals(2, histories.size()); - - for (UpgradeHistoryEntity history : histories) { - assertEquals("ZOOKEEPER", history.getServiceName()); - assertEquals(repoVersionEntity2200, history.getFromReposistoryVersion()); - assertEquals(repoVersionEntity2110, history.getTargetRepositoryVersion()); - } - } - - - /** - * Tests that from/to repository version history is created correctly on the - * upgrade. - * - * @throws Exception - */ - @Test - public void testCreatePatchRevertUpgrade() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - // add a single ZK server and client on 2.1.1.0 - ServiceGroup serviceGroup = cluster.getServiceGroup(UpgradeResourceProvider.DUMMY_SERVICE_GROUP); - Service service = cluster.addService(serviceGroup, "HBASE", "HBASE", repoVersionEntity2110); - ServiceComponent component = service.addServiceComponent("HBASE_MASTER"); - ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - File f = new File("src/test/resources/hbase_version_test.xml"); - repoVersionEntity2112.setType(RepositoryType.PATCH); - repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f))); - repoVersionEntity2112.setVersionXsd("version_definition.xsd"); - repoVersionDao.merge(repoVersionEntity2112); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2112.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgradeEntity = upgrades.get(0); - assertEquals(RepositoryType.PATCH, upgradeEntity.getOrchestration()); - - // should be false since only finalization actually sets this bit - assertEquals(false, upgradeEntity.isRevertAllowed()); - - // fake it now so the rest of the test passes - upgradeEntity.setRevertAllowed(true); - upgradeEntity = upgradeDao.merge(upgradeEntity); - - // !!! make it look like the cluster is done - cluster.setUpgradeEntity(null); - - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REVERT_UPGRADE_ID, upgradeEntity.getId()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(2, upgrades.size()); - - boolean found = false; - - Function function = new Function() { - @Override - public String apply(UpgradeHistoryEntity input) { - return input.getServiceName() + "/" + input.getComponentName(); - }; - }; - - for (UpgradeEntity upgrade : upgrades) { - if (upgrade.getId() != upgradeEntity.getId()) { - found = true; - assertEquals(upgradeEntity.getOrchestration(), upgrade.getOrchestration()); - - Collection upgradeEntityStrings = Collections2.transform(upgradeEntity.getHistory(), function); - Collection upgradeStrings = Collections2.transform(upgrade.getHistory(), function); - - Collection diff = CollectionUtils.disjunction(upgradeEntityStrings, upgradeStrings); - assertEquals("Verify the same set of components was orchestrated", 0, diff.size()); - } - } - - assertTrue(found); - } - - /** - * Tests that when there is no revertable upgrade, a reversion of a specific - * ugprade ID is not allowed. - */ - @Test(expected = SystemException.class) - public void testRevertFailsWhenNoRevertableUpgradeIsFound() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - // add a single ZK server and client on 2.1.1.0 - ServiceGroup serviceGroup = cluster.getServiceGroup(UpgradeResourceProvider.DUMMY_SERVICE_GROUP); - Service service = cluster.addService(serviceGroup, "HBASE", "HBASE", repoVersionEntity2110); - ServiceComponent component = service.addServiceComponent("HBASE_MASTER"); - ServiceComponentHost sch = component.addServiceComponentHost("h1"); - sch.setVersion("2.1.1.0"); - - File f = new File("src/test/resources/hbase_version_test.xml"); - repoVersionEntity2112.setType(RepositoryType.PATCH); - repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f))); - repoVersionEntity2112.setVersionXsd("version_definition.xsd"); - repoVersionDao.merge(repoVersionEntity2112); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID,String.valueOf(repoVersionEntity2112.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - - upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(1, upgrades.size()); - - UpgradeEntity upgradeEntity = upgrades.get(0); - assertEquals(RepositoryType.PATCH, upgradeEntity.getOrchestration()); - - // !!! make it look like the cluster is done - cluster.setUpgradeEntity(null); - - requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REVERT_UPGRADE_ID, upgradeEntity.getId()); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - upgradeResourceProvider.createResources(request); - } - - @Test - public void testCreatePatchWithConfigChanges() throws Exception { - Cluster cluster = clusters.getCluster("c1"); - - File f = new File("src/test/resources/version_definition_test_patch_config.xml"); - repoVersionEntity2112.setType(RepositoryType.PATCH); - repoVersionEntity2112.setVersionXml(IOUtils.toString(new FileInputStream(f))); - repoVersionEntity2112.setVersionXsd("version_definition.xsd"); - repoVersionDao.merge(repoVersionEntity2112); - - List upgrades = upgradeDao.findUpgrades(cluster.getClusterId()); - assertEquals(0, upgrades.size()); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2112.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - // !!! test that a PATCH upgrade skips config changes - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - RequestStatus status = upgradeResourceProvider.createResources(request); - Set resources = status.getAssociatedResources(); - assertEquals(1, resources.size()); - Long requestId = (Long) resources.iterator().next().getPropertyValue("Upgrade/request_id"); - assertNotNull(requestId); - - UpgradeEntity upgradeEntity = upgradeDao.findUpgradeByRequestId(requestId); - assertEquals(RepositoryType.PATCH, upgradeEntity.getOrchestration()); - - HostRoleCommandDAO hrcDAO = injector.getInstance(HostRoleCommandDAO.class); - List commands = hrcDAO.findByRequest(upgradeEntity.getRequestId()); - - boolean foundConfigTask = false; - for (HostRoleCommandEntity command : commands) { - if (StringUtils.isNotBlank(command.getCustomCommandName()) && - command.getCustomCommandName().equals(ConfigureAction.class.getName())) { - foundConfigTask = true; - break; - } - } - assertFalse(foundConfigTask); - - // !!! test that a patch with a supported patch change gets picked up - cluster.setUpgradeEntity(null); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_force_config_change"); - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - - status = upgradeResourceProvider.createResources(request); - resources = status.getAssociatedResources(); - assertEquals(1, resources.size()); - requestId = (Long) resources.iterator().next().getPropertyValue("Upgrade/request_id"); - assertNotNull(requestId); - - upgradeEntity = upgradeDao.findUpgradeByRequestId(requestId); - assertEquals(RepositoryType.PATCH, upgradeEntity.getOrchestration()); - - commands = hrcDAO.findByRequest(upgradeEntity.getRequestId()); - - foundConfigTask = false; - for (HostRoleCommandEntity command : commands) { - if (StringUtils.isNotBlank(command.getCustomCommandName()) && - command.getCustomCommandName().equals(ConfigureAction.class.getName())) { - foundConfigTask = true; - - ExecutionCommandDAO dao = injector.getInstance(ExecutionCommandDAO.class); - ExecutionCommandEntity entity = dao.findByPK(command.getTaskId()); - ExecutionCommandWrapperFactory factory = injector.getInstance(ExecutionCommandWrapperFactory.class); - ExecutionCommandWrapper wrapper = factory.createFromJson(new String(entity.getCommand())); - Map params = wrapper.getExecutionCommand().getCommandParams(); - assertTrue(params.containsKey(ConfigureTask.PARAMETER_ASSOCIATED_SERVICE)); - assertEquals("ZOOKEEPER", params.get(ConfigureTask.PARAMETER_ASSOCIATED_SERVICE)); - - break; - } - } - assertTrue(foundConfigTask); - - - - // !!! test that a regular upgrade will pick up the config change - cluster.setUpgradeEntity(null); - repoVersionEntity2112.setType(RepositoryType.STANDARD); - repoVersionDao.merge(repoVersionEntity2112); - - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - - status = upgradeResourceProvider.createResources(request); - resources = status.getAssociatedResources(); - assertEquals(1, resources.size()); - requestId = (Long) resources.iterator().next().getPropertyValue("Upgrade/request_id"); - assertNotNull(requestId); - - upgradeEntity = upgradeDao.findUpgradeByRequestId(requestId); - assertEquals(RepositoryType.STANDARD, upgradeEntity.getOrchestration()); - - commands = hrcDAO.findByRequest(upgradeEntity.getRequestId()); - - foundConfigTask = false; - for (HostRoleCommandEntity command : commands) { - if (StringUtils.isNotBlank(command.getCustomCommandName()) && - command.getCustomCommandName().equals(ConfigureAction.class.getName())) { - foundConfigTask = true; - break; - } - } - assertTrue(foundConfigTask); - - } - - - - private String parseSingleMessage(String msgStr){ - JsonParser parser = new JsonParser(); - JsonArray msgArray = (JsonArray) parser.parse(msgStr); - JsonObject msg = (JsonObject) msgArray.get(0); - - return msg.get("message").getAsString(); - } - - /** - * Aborts and upgrade. - * - * @param requestId - * @throws Exception - */ - private void abortUpgrade(long requestId) throws Exception { - // now abort the upgrade so another can be created - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_ID, String.valueOf(requestId)); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REQUEST_STATUS, "ABORTED"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SUSPENDED, "false"); - Request request = PropertyHelper.getUpdateRequest(requestProps, null); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - upgradeResourceProvider.updateResources(request, null); - - // !!! this is required since the ActionManager/ActionScheduler isn't - // running and can't remove queued PENDING - it's a cheap way of ensuring - // that the upgrade commands do get aborted - hrcDAO.updateStatusByRequestId(requestId, HostRoleStatus.ABORTED, - HostRoleStatus.IN_PROGRESS_STATUSES); - } - - @Test - public void testTimeouts() throws Exception { - StackEntity stackEntity = stackDAO.find("HDP", "2.1.1"); - RepositoryVersionEntity repoVersionEntity = new RepositoryVersionEntity(); - repoVersionEntity.setDisplayName("My New Version 3"); - List operatingSystems = new ArrayList<>(); - RepoDefinitionEntity repoDefinitionEntity1 = new RepoDefinitionEntity(); - repoDefinitionEntity1.setRepoID("HDP-UTILS"); - repoDefinitionEntity1.setBaseUrl(""); - repoDefinitionEntity1.setRepoName("HDP-UTILS"); - RepoDefinitionEntity repoDefinitionEntity2 = new RepoDefinitionEntity(); - repoDefinitionEntity2.setRepoID("HDP"); - repoDefinitionEntity2.setBaseUrl(""); - repoDefinitionEntity2.setRepoName("HDP"); - RepoOsEntity repoOsEntity = new RepoOsEntity(); - repoOsEntity.setFamily("redhat6"); - repoOsEntity.setAmbariManaged(true); - repoOsEntity.addRepoDefinition(repoDefinitionEntity1); - repoOsEntity.addRepoDefinition(repoDefinitionEntity2); - operatingSystems.add(repoOsEntity); - - repoVersionEntity.addRepoOsEntities(operatingSystems); - repoVersionEntity.setStack(stackEntity); - repoVersionEntity.setVersion("2.2.2.3"); - repoVersionDao.create(repoVersionEntity); - - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - RequestStatus status = upgradeResourceProvider.createResources(request); - - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - - ActionManager am = injector.getInstance(ActionManager.class); - - List commands = am.getRequestTasks(id); - - boolean found = false; - - for (HostRoleCommand command : commands) { - ExecutionCommandWrapper wrapper = command.getExecutionCommandWrapper(); - - if (command.getRole().equals(Role.ZOOKEEPER_SERVER) && command.getRoleCommand().equals(RoleCommand.CUSTOM_COMMAND)) { - Map commandParams = wrapper.getExecutionCommand().getCommandParams(); - assertTrue(commandParams.containsKey(KeyNames.COMMAND_TIMEOUT)); - assertEquals("824",commandParams.get(KeyNames.COMMAND_TIMEOUT)); - found = true; - } - } - - assertTrue("ZooKeeper timeout override was found", found); - - } - - /** - * Tests that commands created for {@link org.apache.ambari.server.state.stack.upgrade.StageWrapper.Type#UPGRADE_TASKS} set the - * service and component on the {@link ExecutionCommand}. - *

- * Without this, commands of this type would not be able to determine which - * service/component repository they should use when the command is scheduled - * to run. - * - * @throws Exception - */ - @Test - public void testExecutionCommandServiceAndComponent() throws Exception { - Map requestProps = new HashMap<>(); - requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); - requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); - requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_execute_task_test"); - requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); - requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - - ResourceProvider upgradeResourceProvider = createProvider(amc); - - Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); - RequestStatus status = upgradeResourceProvider.createResources(request); - - Set createdResources = status.getAssociatedResources(); - assertEquals(1, createdResources.size()); - Resource res = createdResources.iterator().next(); - Long id = (Long) res.getPropertyValue("Upgrade/request_id"); - assertNotNull(id); - assertEquals(Long.valueOf(1), id); - - - ActionManager am = injector.getInstance(ActionManager.class); - List commands = am.getRequestTasks(id); - - boolean foundActionExecuteCommand = false; - for (HostRoleCommand command : commands) { - ExecutionCommand executionCommand = command.getExecutionCommandWrapper().getExecutionCommand(); - if (StringUtils.equals(UpgradeResourceProvider.EXECUTE_TASK_ROLE, - executionCommand.getRole())) { - foundActionExecuteCommand = true; - assertNotNull(executionCommand.getServiceName()); - assertNotNull(executionCommand.getComponentName()); - } - } - - assertTrue( - "There was no task found with the role of " + UpgradeResourceProvider.EXECUTE_TASK_ROLE, - foundActionExecuteCommand); - } - - /** - * - */ - private class MockModule implements Module { - /** - * - */ - @Override - public void configure(Binder binder) { - binder.bind(ConfigHelper.class).toInstance(configHelper); - } - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java index f94cb12985f..473a7dd1762 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeSummaryResourceProviderTest.java @@ -174,7 +174,7 @@ public void createCluster() throws Exception { clusters.mapHostToCluster("h1", "c1"); // add a single ZOOKEEPER server - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service service = cluster.addService(serviceGroup, "ZOOKEEPER", "ZOOKEEPER", repoVersionEntity); ServiceComponent component = service.addServiceComponent("ZOOKEEPER_SERVER"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java index c3052e46ff8..0c9805c6207 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/VersionDefinitionResourceProviderTest.java @@ -698,7 +698,7 @@ private void makeService(String serviceName, RepositoryVersionEntity serviceRepo } catch (AmbariException e) { clusters.addCluster(clusterName, parentEntity.getStackId()); cluster = clusters.getCluster(clusterName); - serviceGroup = cluster.addServiceGroup(serviceGroupName); + serviceGroup = cluster.addServiceGroup(serviceGroupName, "HDP-1.0"); } cluster.addService(serviceGroup, serviceName, serviceName, serviceRepo); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/GeneralServiceCalculatedStateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/GeneralServiceCalculatedStateTest.java index 720a1f88850..7a4637640b0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/GeneralServiceCalculatedStateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/utilities/state/GeneralServiceCalculatedStateTest.java @@ -87,7 +87,7 @@ public void configure(Binder binder) { clusters.addCluster(clusterName, stack211); cluster = clusters.getCluster(clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stack211.getStackId()); service = cluster.addService(serviceGroup, getServiceName(), getServiceName(), repositoryVersion); createComponentsAndHosts(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/EventsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/EventsTest.java index bfe0b51629f..a312f8efbde 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/EventsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/EventsTest.java @@ -121,7 +121,7 @@ public void setup() throws Exception { m_cluster = m_clusters.getCluster(m_clusterName); Assert.assertNotNull(m_cluster); - serviceGroup = m_cluster.addServiceGroup("CORE"); + serviceGroup = m_cluster.addServiceGroup("CORE", stackId.getStackId()); m_cluster.setDesiredStackVersion(stackId); m_repositoryVersion = m_helper.getOrCreateRepositoryVersion(stackId, REPO_VERSION); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java index 985facf5274..4f05d9ba752 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java @@ -151,7 +151,7 @@ private RepositoryVersionEntity createClusterAndHosts(String INSTALLED_VERSION, Map> zkTopology = new HashMap<>(); List zkServerHosts = Arrays.asList(0, 1, 2); zkTopology.put("ZOOKEEPER_SERVER", new ArrayList<>(zkServerHosts)); - ServiceGroup serviceGroup = c1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c1.addServiceGroup("CORE", "HDP-1.0"); addService(c1, serviceGroup, hostList, zkTopology, "ZOOKEEPER", repositoryVersionEntity); // install new version @@ -477,7 +477,7 @@ public void testComponentHostVersionNotRequired() throws Exception { .put("NAMENODE", Lists.newArrayList(0)) .put("DATANODE", Lists.newArrayList(1)) .build(); - ServiceGroup serviceGroup = c1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c1.addServiceGroup("CORE", "HDP-1.0"); addService(c1, serviceGroup, allHosts, topology, "HDFS", repo); topology = new ImmutableMap.Builder>() diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java deleted file mode 100644 index c4b1f0b4d4f..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/StackUpgradeFinishListenerTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.events.listeners.upgrade; - -import static org.easymock.EasyMock.expect; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.events.StackUpgradeFinishEvent; -import org.apache.ambari.server.events.publishers.VersionEventPublisher; -import org.apache.ambari.server.metadata.RoleCommandOrderProvider; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.UpgradeEntity; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentHost; -import org.easymock.EasyMockRunner; -import org.easymock.EasyMockSupport; -import org.easymock.Mock; -import org.easymock.MockType; -import org.easymock.TestSubject; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -import com.google.inject.Provider; - - -/** - * StackVersionListener tests. - */ -@RunWith(EasyMockRunner.class) -public class StackUpgradeFinishListenerTest extends EasyMockSupport { - - private static final String INVALID_NEW_VERSION = "1.2.3.4-5678"; - private static final String VALID_NEW_VERSION = "2.4.0.0-1000"; - private static final String SERVICE_COMPONENT_NAME = "Some component name"; - private static final String SERVICE_NAME = "Service name"; - private static final Long CLUSTER_ID = 1L; - private static final String UNKNOWN_VERSION = "UNKNOWN"; - private static final String VALID_PREVIOUS_VERSION = "2.2.0.0"; - private static final RepositoryVersionEntity DUMMY_REPOSITORY_VERSION_ENTITY = new RepositoryVersionEntity(); - private static final UpgradeEntity DUMMY_UPGRADE_ENTITY = new UpgradeEntity(); - public static final String STACK_NAME = "HDP-2.4.0.0"; - public static final String STACK_VERSION = "2.4.0.0"; - - private Cluster cluster; - private ServiceComponentHost sch; - private Service service; - private ServiceComponent serviceComponent; - private VersionEventPublisher publisher = new VersionEventPublisher(); - - @TestSubject - private StackUpgradeFinishListener listener = new StackUpgradeFinishListener(publisher); - - @Mock(type = MockType.NICE) - private Provider roleCommandOrderProviderProviderMock; - - @Before - public void setup() throws Exception { - cluster = createNiceMock(Cluster.class); - serviceComponent = createNiceMock(ServiceComponent.class); - service = createNiceMock(Service.class); - Map services = new HashMap<>(); - services.put("mock_service",service); - Map components = new HashMap<>(); - components.put("mock_component", serviceComponent); - - expect(cluster.getServices()).andReturn(services); - expect(service.getServiceComponents()).andReturn(components); - serviceComponent.updateComponentInfo(); - service.updateServiceInfo(); - } - - @Test - public void testupdateComponentInfo() throws AmbariException { - replayAll(); - - sendEventAndVerify(); - } - - - private void sendEventAndVerify() { - StackUpgradeFinishEvent event = new StackUpgradeFinishEvent(cluster); - listener.onAmbariEvent(event); - - verifyAll(); - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java index 298846443eb..3be8333122b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/OrmTestHelper.java @@ -206,6 +206,7 @@ public void createDefaultData() { ServiceGroupEntity serviceGroupEntity = new ServiceGroupEntity(); serviceGroupEntity.setServiceGroupName(SERVICE_GROUP_NAME); serviceGroupEntity.setClusterEntity(clusterEntity); + serviceGroupEntity.setStack(stackEntity); ClusterServiceEntity clusterServiceEntity = new ClusterServiceEntity(); clusterServiceEntity.setServiceType("HDFS"); @@ -415,7 +416,7 @@ public Cluster buildNewCluster(Clusters clusters, clusters.addCluster(clusterName, stackId); Cluster cluster = clusters.getCluster(clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup(SERVICE_GROUP_NAME); + ServiceGroup serviceGroup = cluster.addServiceGroup(SERVICE_GROUP_NAME, stackId.getStackId()); cluster = initializeClusterWithStack(cluster); addHost(clusters, cluster, hostName); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertDispatchDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertDispatchDAOTest.java index 623eeb0a445..c20ce18758a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertDispatchDAOTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertDispatchDAOTest.java @@ -117,7 +117,7 @@ public void setup() throws Exception { EventBusSynchronizer.synchronizeAmbariEventPublisher(m_injector); m_cluster = m_clusters.getClusterById(m_helper.createCluster()); - serviceGroup = m_cluster.addServiceGroup("CORE"); + serviceGroup = m_cluster.addServiceGroup("CORE", "HDP-1.0"); m_helper.initializeClusterWithStack(m_cluster); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java index 60d7a246b59..2d7826bf4a4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/AlertsDAOTest.java @@ -123,7 +123,7 @@ public void setup() throws Exception { // install YARN so there is at least 1 service installed and no // unexpected alerts since the test YARN service doesn't have any alerts m_cluster = m_clusters.getClusterById(m_helper.createCluster()); - serviceGroup = m_cluster.addServiceGroup("CORE"); + serviceGroup = m_cluster.addServiceGroup("CORE", "HDP-1.0"); m_helper.initializeClusterWithStack(m_cluster); m_helper.addHost(m_clusters, m_cluster, HOSTNAME); m_helper.installYarnService(m_cluster, m_serviceFactory, diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/MpackDAOTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/MpackDAOTest.java index 64dae36e53b..cc36b48ee9a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/MpackDAOTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/dao/MpackDAOTest.java @@ -56,7 +56,7 @@ public void testCreateFind() { // create 2 definitions for (int i = 1; i < 3; i++) { MpackEntity definition = new MpackEntity(); - definition.setMpackId(new Long(100)+i); + definition.setId(new Long(100)+i); definition.setMpackName("testMpack" + i); definition.setRegistryId(Long.valueOf(i)); definition.setMpackVersion("3.0.0.0-12"+i); @@ -70,7 +70,7 @@ public void testCreateFind() { assertEquals(2, definitions.size()); definitions = m_dao.findByNameVersion("testMpack1","3.0.0.0-121"); assertEquals(1, definitions.size()); - assertEquals(new Long(101),(Long)definitions.get(0).getMpackId()); + assertEquals(new Long(101),(Long)definitions.get(0).getId()); MpackEntity entity = m_dao.findById(new Long(102)); assertEquals(entity.getMpackName(),"testMpack2"); assertEquals(entity.getMpackVersion(),"3.0.0.0-122"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/MpackEntityTest.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/MpackEntityTest.java index 87ac934e209..d2d2476b022 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/MpackEntityTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/MpackEntityTest.java @@ -38,12 +38,12 @@ public void testHashCodeAndEquals(){ Assert.assertEquals(entity1.hashCode(), entity2.hashCode()); Assert.assertTrue(Objects.equals(entity1, entity2)); - entity1.setMpackId(new Long(1)); - entity2.setMpackId(new Long(2)); + entity1.setId(new Long(1)); + entity2.setId(new Long(2)); Assert.assertNotSame(entity1.hashCode(), entity2.hashCode()); Assert.assertFalse(Objects.equals(entity1, entity2)); - entity2.setMpackId(new Long(1)); + entity2.setId(new Long(1)); Assert.assertEquals(entity1.hashCode(), entity2.hashCode()); Assert.assertTrue(Objects.equals(entity1, entity2)); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java index 08df6c2dd79..40ca1190338 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ComponentVersionCheckActionTest.java @@ -211,7 +211,7 @@ private void makeUpgradeCluster(StackId sourceStack, String sourceRepo, StackId upgradeEntity.setRequestEntity(requestEntity); upgradeEntity.setUpgradePackage(""); upgradeEntity.setRepositoryVersion(toRepositoryVersion); - upgradeEntity.setUpgradeType(UpgradeType.NON_ROLLING); + upgradeEntity.setUpgradeType(UpgradeType.EXPRESS); upgradeDAO.create(upgradeEntity); c.setUpgradeEntity(upgradeEntity); @@ -276,7 +276,7 @@ private void makeCrossStackUpgradeCluster(StackId sourceStack, String sourceRepo upgradeEntity.setRequestEntity(requestEntity); upgradeEntity.setUpgradePackage(""); upgradeEntity.setRepositoryVersion(toRepositoryVersion); - upgradeEntity.setUpgradeType(UpgradeType.NON_ROLLING); + upgradeEntity.setUpgradeType(UpgradeType.EXPRESS); upgradeDAO.create(upgradeEntity); c.setUpgradeEntity(upgradeEntity); @@ -340,7 +340,7 @@ public void testMixedComponentVersions() throws Exception { Clusters clusters = m_injector.getInstance(Clusters.class); Cluster cluster = clusters.getCluster("c1"); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", targetStack.getStackId()); RepositoryVersionEntity sourceRepoVersion = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_0); RepositoryVersionEntity targetRepoVersion = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_1_0); @@ -441,7 +441,7 @@ public void testMatchingPartialVersions() throws Exception { host.setOsInfo("redhat6"); Cluster cluster = clusters.getCluster("c1"); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", targetStack.getStackId()); clusters.mapHostToCluster("h1", "c1"); RepositoryVersionEntity repositoryVersion2110 = m_helper.getOrCreateRepositoryVersion( diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java deleted file mode 100644 index 2c8cd913747..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java +++ /dev/null @@ -1,1854 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.serveraction.upgrades; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.persistence.EntityManager; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.ServiceComponentNotFoundException; -import org.apache.ambari.server.ServiceNotFoundException; -import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; -import org.apache.ambari.server.actionmanager.HostRoleCommand; -import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; -import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.OrmTestHelper; -import org.apache.ambari.server.orm.dao.RequestDAO; -import org.apache.ambari.server.orm.dao.UpgradeDAO; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.RequestEntity; -import org.apache.ambari.server.orm.entities.UpgradeEntity; -import org.apache.ambari.server.orm.entities.UpgradeHistoryEntity; -import org.apache.ambari.server.serveraction.ServerAction; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigFactory; -import org.apache.ambari.server.state.ConfigHelper; -import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentFactory; -import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceComponentHostFactory; -import org.apache.ambari.server.state.ServiceFactory; -import org.apache.ambari.server.state.ServiceGroup; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.State; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Insert; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.InsertType; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Replace; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.Transfer; -import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; -import org.apache.ambari.server.state.stack.upgrade.PropertyKeyState; -import org.apache.ambari.server.state.stack.upgrade.TransferCoercionType; -import org.apache.ambari.server.state.stack.upgrade.TransferOperation; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; -import org.apache.commons.lang3.StringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.gson.Gson; -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; - -/** - * Tests upgrade-related server side actions - */ -public class ConfigureActionTest { - - @Inject - private Injector m_injector; - - @Inject - private OrmTestHelper m_helper; - - @Inject - private HostRoleCommandFactory hostRoleCommandFactory; - - @Inject - private ServiceFactory serviceFactory; - - @Inject - private ConfigHelper m_configHelper; - - @Inject - private Clusters clusters; - - @Inject - private ConfigFactory configFactory; - - @Inject - private ConfigureAction action; - - @Inject - private RequestDAO requestDAO; - - @Inject - private UpgradeDAO upgradeDAO; - - @Inject - private ServiceComponentFactory serviceComponentFactory; - - @Inject - private ServiceComponentHostFactory serviceComponentHostFactory; - - private RepositoryVersionEntity repoVersion2110; - private RepositoryVersionEntity repoVersion2111; - private RepositoryVersionEntity repoVersion2200; - - private final Map> NO_ATTRIBUTES = new HashMap<>(); - - @Before - public void setup() throws Exception { - m_injector = Guice.createInjector(new InMemoryDefaultTestModule()); - m_injector.getInstance(GuiceJpaInitializer.class); - m_injector.injectMembers(this); - - repoVersion2110 = m_helper.getOrCreateRepositoryVersion(new StackId("HDP-2.1.1"), "2.1.1.0-1234"); - repoVersion2111 = m_helper.getOrCreateRepositoryVersion(new StackId("HDP-2.1.1"), "2.1.1.1-5678"); - repoVersion2200 = m_helper.getOrCreateRepositoryVersion(new StackId("HDP-2.2.0"), "2.2.0.0-1234"); - - makeUpgradeCluster(); - } - - @After - public void teardown() throws Exception { - H2DatabaseCleaner.clearDatabase(m_injector.getProvider(EntityManager.class).get()); - } - - - /** - * Tests that a new configuration is created when upgrading across stack when - * there is no existing configuration with the correct target stack. - * - * @throws Exception - */ - @Test - public void testNewConfigCreatedWhenUpgradingAcrossStacks() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse(StringUtils.equals("version2", config.getTag())); - assertEquals("11", config.getProperties().get("initLimit")); - } - - /** - * Tests that if a configuration with the target stack already exists, then it - * will be re-used instead of a new one created. - * - * @throws Exception - */ - @Test - public void testConfigurationWithTargetStackUsed() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - } - }; - - Config config = createConfig(c, repoVersion2200, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("version2", config.getTag()); - assertEquals("11", config.getProperties().get("initLimit")); - } - - /** - * Tests that DELETE "*" with edit preserving works correctly. - * - * @throws Exception - */ - @Test - public void testDeletePreserveChanges() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - // create a config for zoo.cfg with two values; one is a stack value and the - // other is custom - Map properties = new HashMap() { - { - put("tickTime", "2000"); - put("foo", "bar"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - - createUpgrade(c, repoVersion2111); - - // delete all keys, preserving edits or additions - List transfers = new ArrayList<>(); - Transfer transfer = new Transfer(); - transfer.operation = TransferOperation.DELETE; - transfer.deleteKey = "*"; - transfer.preserveEdits = true; - transfers.add(transfer); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - // make sure there are now 3 versions after the merge - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - // time to check our values; there should only be 1 left since tickTime was - // removed - Map map = config.getProperties(); - assertEquals("bar", map.get("foo")); - assertFalse(map.containsKey("tickTime")); - } - - @Test - public void testConfigTransferCopy() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("copyIt", "10"); - put("moveIt", "10"); - put("deleteIt", "10"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - // normal copy - List transfers = new ArrayList<>(); - Transfer transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.fromKey = "copyIt"; - transfer.toKey = "copyKey"; - transfers.add(transfer); - - // copy with default - transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.fromKey = "copiedFromMissingKeyWithDefault"; - transfer.toKey = "copiedToMissingKeyWithDefault"; - transfer.defaultValue = "defaultValue"; - transfers.add(transfer); - - // normal move - transfer = new Transfer(); - transfer.operation = TransferOperation.MOVE; - transfer.fromKey = "moveIt"; - transfer.toKey = "movedKey"; - transfers.add(transfer); - - // move with default - transfer = new Transfer(); - transfer.operation = TransferOperation.MOVE; - transfer.fromKey = "movedFromKeyMissingWithDefault"; - transfer.toKey = "movedToMissingWithDefault"; - transfer.defaultValue = "defaultValue2"; - transfer.mask = true; - transfers.add(transfer); - - transfer = new Transfer(); - transfer.operation = TransferOperation.DELETE; - transfer.deleteKey = "deleteIt"; - transfers.add(transfer); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals("11", map.get("initLimit")); - assertEquals("10", map.get("copyIt")); - assertTrue(map.containsKey("copyKey")); - assertEquals(map.get("copyIt"), map.get("copyKey")); - assertFalse(map.containsKey("moveIt")); - assertTrue(map.containsKey("movedKey")); - assertFalse(map.containsKey("deletedKey")); - assertTrue(map.containsKey("copiedToMissingKeyWithDefault")); - assertEquals("defaultValue", map.get("copiedToMissingKeyWithDefault")); - assertTrue(map.containsKey("movedToMissingWithDefault")); - assertEquals("defaultValue2", map.get("movedToMissingWithDefault")); - - transfers.clear(); - transfer = new Transfer(); - transfer.operation = TransferOperation.DELETE; - transfer.deleteKey = "*"; - transfer.preserveEdits = true; - transfer.keepKeys.add("copyKey"); - // The below key should be ignored/not added as it doesn't exist originally as part of transfer. - transfer.keepKeys.add("keyNotExisting"); - // The 'null' passed as part of key should be ignored as part of transfer operation. - transfer.keepKeys.add(null); - - - transfers.add(transfer); - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - report = action.execute(null); - assertNotNull(report); - - assertEquals(4, c.getConfigsByType("zoo.cfg").size()); - config = c.getDesiredConfigByType("zoo.cfg"); - map = config.getProperties(); - assertEquals(6, map.size()); - assertTrue(map.containsKey("initLimit")); // it just changed to 11 from 10 - assertTrue(map.containsKey("copyKey")); // is new - // Below two keys should not have been added in the map. - assertFalse(map.containsKey("keyNotExisting")); - assertFalse(map.containsKey(null)); - } - - @Test - public void testCoerceValueOnCopy() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("zoo.server.csv", "c6401,c6402, c6403"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - - // copy with coerce - List transfers = new ArrayList<>(); - Transfer transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.coerceTo = TransferCoercionType.YAML_ARRAY; - transfer.fromKey = "zoo.server.csv"; - transfer.toKey = "zoo.server.array"; - transfer.defaultValue = "['foo','bar']"; - transfers.add(transfer); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals("c6401,c6402, c6403", map.get("zoo.server.csv")); - assertEquals("['c6401','c6402','c6403']", map.get("zoo.server.array")); - } - - @Test - public void testValueReplacement() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("key_to_replace", "My New Cat"); - put("key_with_no_match", "WxyAndZ"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - - // Replacement task - List replacements = new ArrayList<>(); - Replace replace = new Replace(); - replace.key = "key_to_replace"; - replace.find = "New Cat"; - replace.replaceWith = "Wet Dog"; - replacements.add(replace); - - replace = new Replace(); - replace.key = "key_with_no_match"; - replace.find = "abc"; - replace.replaceWith = "def"; - replacements.add(replace); - - commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, new Gson().toJson(replacements)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - assertEquals("My Wet Dog", config.getProperties().get("key_to_replace")); - assertEquals("WxyAndZ", config.getProperties().get("key_with_no_match")); - } - - /** - * Tests that replacing a {@code null} value works. - * - * @throws Exception - */ - @Test - public void testValueReplacementWithMissingConfigurations() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("existing", "This exists!"); - put("missing", null); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - - // Replacement task - List replacements = new ArrayList<>(); - Replace replace = new Replace(); - replace.key = "missing"; - replace.find = "foo"; - replace.replaceWith = "bar"; - replacements.add(replace); - - commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, new Gson().toJson(replacements)); - - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName("c1"); - executionCommand.setRoleParams(new HashMap<>()); - executionCommand.getRoleParams().put(ServerAction.ACTION_USER_NAME, "username"); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertEquals(null, config.getProperties().get("missing")); - } - - @Test - public void testMultipleKeyValuesPerTask() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("fooKey", "barValue"); - } - }; - - Config config = createConfig(c, repoVersion2200, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - // create several configurations - List configurations = new ArrayList<>(); - ConfigurationKeyValue fooKey2 = new ConfigurationKeyValue(); - configurations.add(fooKey2); - fooKey2.key = "fooKey2"; - fooKey2.value = "barValue2"; - - ConfigurationKeyValue fooKey3 = new ConfigurationKeyValue(); - configurations.add(fooKey3); - fooKey3.key = "fooKey3"; - fooKey3.value = "barValue3"; - fooKey3.mask = true; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("barValue", config.getProperties().get("fooKey")); - assertEquals("barValue2", config.getProperties().get("fooKey2")); - assertEquals("barValue3", config.getProperties().get("fooKey3")); - assertTrue(report.getStdOut().contains("******")); - - } - - @Test - public void testAllowedSet() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("set.key.1", "s1"); - put("set.key.2", "s2"); - put("set.key.3", "s3"); - put("set.key.4", "s4"); - } - }; - - Config config = createConfig(c, repoVersion2200, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - // create several configurations - List configurations = new ArrayList<>(); - ConfigurationKeyValue fooKey1 = new ConfigurationKeyValue(); - configurations.add(fooKey1); - fooKey1.key = "fooKey1"; - fooKey1.value = "barValue1"; - - ConfigurationKeyValue fooKey2 = new ConfigurationKeyValue(); - configurations.add(fooKey2); - fooKey2.key = "fooKey2"; - fooKey2.value = "barValue2"; - - ConfigurationKeyValue fooKey3 = new ConfigurationKeyValue(); - configurations.add(fooKey3); - fooKey3.key = "fooKey3"; - fooKey3.value = "barValue3"; - fooKey3.ifKey = "set.key.1"; - fooKey3.ifType = "zoo.cfg"; - fooKey3.ifValue = "s1"; - - ConfigurationKeyValue fooKey4 = new ConfigurationKeyValue(); - configurations.add(fooKey4); - fooKey4.key = "fooKey4"; - fooKey4.value = "barValue4"; - fooKey4.ifKey = "set.key.2"; - fooKey4.ifType = "zoo.cfg"; - fooKey4.ifKeyState= PropertyKeyState.PRESENT; - - ConfigurationKeyValue fooKey5 = new ConfigurationKeyValue(); - configurations.add(fooKey5); - fooKey5.key = "fooKey5"; - fooKey5.value = "barValue5"; - fooKey5.ifKey = "abc"; - fooKey5.ifType = "zoo.cfg"; - fooKey5.ifKeyState= PropertyKeyState.ABSENT; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("barValue1", config.getProperties().get("fooKey1")); - assertEquals("barValue2", config.getProperties().get("fooKey2")); - assertEquals("barValue3", config.getProperties().get("fooKey3")); - assertEquals("barValue4", config.getProperties().get("fooKey4")); - assertEquals("barValue5", config.getProperties().get("fooKey5")); - assertEquals("s1", config.getProperties().get("set.key.1")); - assertEquals("s2", config.getProperties().get("set.key.2")); - assertEquals("s3", config.getProperties().get("set.key.3")); - assertEquals("s4", config.getProperties().get("set.key.4")); - } - - @Test - public void testDisallowedSet() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("set.key.1", "s1"); - put("set.key.2", "s2"); - put("set.key.3", "s3"); - put("set.key.4", "s4"); - } - }; - - Config config = createConfig(c, repoVersion2200, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - // create several configurations - List configurations = new ArrayList<>(); - ConfigurationKeyValue fooKey3 = new ConfigurationKeyValue(); - configurations.add(fooKey3); - fooKey3.key = "fooKey3"; - fooKey3.value = "barValue3"; - fooKey3.ifKey = "set.key.1"; - fooKey3.ifType = "zoo.cfg"; - fooKey3.ifValue = "no-such-value"; - - ConfigurationKeyValue fooKey4 = new ConfigurationKeyValue(); - configurations.add(fooKey4); - fooKey4.key = "fooKey4"; - fooKey4.value = "barValue4"; - fooKey4.ifKey = "set.key.2"; - fooKey4.ifType = "zoo.cfg"; - fooKey4.ifKeyState= PropertyKeyState.ABSENT; - - ConfigurationKeyValue fooKey5 = new ConfigurationKeyValue(); - configurations.add(fooKey5); - fooKey5.key = "fooKey5"; - fooKey5.value = "barValue5"; - fooKey5.ifKey = "abc"; - fooKey5.ifType = "zoo.cfg"; - fooKey5.ifKeyState= PropertyKeyState.PRESENT; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("s1", config.getProperties().get("set.key.1")); - assertEquals("s2", config.getProperties().get("set.key.2")); - assertEquals("s3", config.getProperties().get("set.key.3")); - assertEquals("s4", config.getProperties().get("set.key.4")); - assertFalse(config.getProperties().containsKey("fooKey3")); - assertFalse(config.getProperties().containsKey("fooKey4")); - assertFalse(config.getProperties().containsKey("fooKey5")); - } - - @Test - public void testAllowedReplacment() throws Exception { - - - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("replace.key.1", "r1"); - put("replace.key.2", "r2"); - put("replace.key.3", "r3a1"); - put("replace.key.4", "r4"); - put("replace.key.5", "r5"); - } - }; - - Config config = createConfig(c, repoVersion2200, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - // create several configurations - List replacements = new ArrayList<>(); - Replace replace = new Replace(); - replace.key = "replace.key.3"; - replace.find = "a"; - replace.replaceWith = "A"; - replacements.add(replace); - - Replace replace2 = new Replace(); - replacements.add(replace2); - replace2.key = "replace.key.4"; - replace2.find = "r"; - replace2.replaceWith = "R"; - replace2.ifKey = "replace.key.1"; - replace2.ifType = "zoo.cfg"; - replace2.ifValue = "r1"; - replacements.add(replace2); - - Replace replace3 = new Replace(); - replacements.add(replace3); - replace3.key = "replace.key.2"; - replace3.find = "r"; - replace3.replaceWith = "R"; - replace3.ifKey = "replace.key.1"; - replace3.ifType = "zoo.cfg"; - replace3.ifKeyState = PropertyKeyState.PRESENT; - replacements.add(replace3); - - Replace replace4 = new Replace(); - replacements.add(replace3); - replace4.key = "replace.key.5"; - replace4.find = "r"; - replace4.replaceWith = "R"; - replace4.ifKey = "no.such.key"; - replace4.ifType = "zoo.cfg"; - replace4.ifKeyState = PropertyKeyState.ABSENT; - replacements.add(replace4); - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, new Gson().toJson(replacements)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("r1", config.getProperties().get("replace.key.1")); - assertEquals("R2", config.getProperties().get("replace.key.2")); - assertEquals("r3A1", config.getProperties().get("replace.key.3")); - assertEquals("R4", config.getProperties().get("replace.key.4")); - assertEquals("R5", config.getProperties().get("replace.key.5")); - } - - @Test - public void testDisallowedReplacment() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("replace.key.1", "r1"); - put("replace.key.2", "r2"); - put("replace.key.3", "r3a1"); - put("replace.key.4", "r4"); - put("replace.key.5", "r5"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - // create several configurations - List replacements = new ArrayList<>(); - - Replace replace2 = new Replace(); - replacements.add(replace2); - replace2.key = "replace.key.4"; - replace2.find = "r"; - replace2.replaceWith = "R"; - replace2.ifKey = "replace.key.1"; - replace2.ifType = "zoo.cfg"; - replace2.ifValue = "not-this-value"; - replacements.add(replace2); - - Replace replace3 = new Replace(); - replacements.add(replace3); - replace3.key = "replace.key.2"; - replace3.find = "r"; - replace3.replaceWith = "R"; - replace3.ifKey = "replace.key.1"; - replace3.ifType = "zoo.cfg"; - replace3.ifKeyState = PropertyKeyState.ABSENT; - replacements.add(replace3); - - Replace replace4 = new Replace(); - replacements.add(replace3); - replace4.key = "replace.key.5"; - replace4.find = "r"; - replace4.replaceWith = "R"; - replace4.ifKey = "no.such.key"; - replace4.ifType = "zoo.cfg"; - replace4.ifKeyState = PropertyKeyState.PRESENT; - replacements.add(replace4); - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_REPLACEMENTS, new Gson().toJson(replacements)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertEquals("r1", config.getProperties().get("replace.key.1")); - assertEquals("r2", config.getProperties().get("replace.key.2")); - assertEquals("r3a1", config.getProperties().get("replace.key.3")); - assertEquals("r4", config.getProperties().get("replace.key.4")); - assertEquals("r5", config.getProperties().get("replace.key.5")); - } - - @Test - public void testAllowedTransferCopy() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("copy.key.1", "c1"); - put("copy.key.2", "c2"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2200); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - // normal copy - List transfers = new ArrayList<>(); - Transfer transfer1 = new Transfer(); - transfer1.operation = TransferOperation.COPY; - transfer1.fromKey = "copy.key.1"; - transfer1.toKey = "copy.to.key.1"; - transfers.add(transfer1); - - // copy with default - Transfer transfer2 = new Transfer(); - transfer2.operation = TransferOperation.COPY; - transfer2.fromKey = "copy.key.no.need.to.exit.1"; - transfer2.toKey = "copy.to.key.with.default.1"; - transfer2.defaultValue = "defaultValue"; - transfers.add(transfer2); - - Transfer transfer3 = new Transfer(); - transfer3.operation = TransferOperation.COPY; - transfer3.fromKey = "copy.key.2"; - transfer3.toKey = "copy.to.key.2"; - transfer3.ifKey = "initLimit"; - transfer3.ifType = "zoo.cfg"; - transfer3.ifValue = "10"; - transfers.add(transfer3); - - Transfer transfer4 = new Transfer(); - transfer4.operation = TransferOperation.COPY; - transfer4.fromKey = "copy.key.2"; - transfer4.toKey = "copy.to.key.3"; - transfer4.ifKey = "initLimit"; - transfer4.ifType = "zoo.cfg"; - transfer4.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer4); - - Transfer transfer5 = new Transfer(); - transfer5.operation = TransferOperation.COPY; - transfer5.fromKey = "copy.key.no.need.to.exist.2"; - transfer5.toKey = "copy.to.key.with.default.2"; - transfer5.defaultValue = "defaultValue2"; - transfer5.ifKey = "no.such.key"; - transfer5.ifType = "zoo.cfg"; - transfer5.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer5); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(8, map.size()); - assertEquals("11", map.get("initLimit")); - assertEquals(map.get("copy.key.1"), map.get("copy.to.key.1")); - assertTrue(!map.containsKey("copy.key.no.need.to.exit.1")); - assertEquals("defaultValue", map.get("copy.to.key.with.default.1")); - assertTrue(!map.containsKey("copy.key.no.need.to.exit.2")); - assertEquals("defaultValue2", map.get("copy.to.key.with.default.2")); - assertEquals(map.get("copy.key.2"), map.get("copy.to.key.2")); - assertEquals(map.get("copy.key.2"), map.get("copy.to.key.3")); - } - - @Test - public void testDisallowedTransferCopy() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("copy.key.1", "c1"); - put("copy.key.2", "c2"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - List transfers = new ArrayList<>(); - Transfer transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.fromKey = "copy.key.2"; - transfer.toKey = "copy.to.key.2"; - transfer.ifKey = "initLimit"; - transfer.ifType = "zoo.cfg"; - transfer.ifValue = "not-the-real-value"; - transfers.add(transfer); - - transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.fromKey = "copy.key.2"; - transfer.toKey = "copy.to.key.3"; - transfer.ifKey = "initLimit"; - transfer.ifType = "zoo.cfg"; - transfer.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer); - - transfer = new Transfer(); - transfer.operation = TransferOperation.COPY; - transfer.fromKey = "copy.key.no.need.to.exist.2"; - transfer.toKey = "copy.to.key.with.default.2"; - transfer.defaultValue = "defaultValue2"; - transfer.ifKey = "no.such.key"; - transfer.ifType = "zoo.cfg"; - transfer.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName("c1"); - executionCommand.setRoleParams(new HashMap<>()); - executionCommand.getRoleParams().put(ServerAction.ACTION_USER_NAME, "username"); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(3, map.size()); - assertEquals("11", map.get("initLimit")); - assertEquals("c1", map.get("copy.key.1")); - assertEquals("c2", map.get("copy.key.2")); - } - - @Test - public void testAllowedTransferMove() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("move.key.1", "m1"); - put("move.key.2", "m2"); - put("move.key.3", "m3"); - put("move.key.4", "m4"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - List transfers = new ArrayList<>(); - Transfer transfer1 = new Transfer(); - transfer1.operation = TransferOperation.MOVE; - transfer1.fromKey = "move.key.1"; - transfer1.toKey = "move.to.key.1"; - transfers.add(transfer1); - - Transfer transfer2 = new Transfer(); - transfer2.operation = TransferOperation.MOVE; - transfer2.fromKey = "move.key.2"; - transfer2.toKey = "move.to.key.2"; - transfer2.ifKey = "initLimit"; - transfer2.ifType = "zoo.cfg"; - transfer2.ifValue = "10"; - transfers.add(transfer2); - - Transfer transfer3 = new Transfer(); - transfer3.operation = TransferOperation.MOVE; - transfer3.fromKey = "move.key.3"; - transfer3.toKey = "move.to.key.3"; - transfer3.ifKey = "initLimit"; - transfer3.ifType = "zoo.cfg"; - transfer3.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer3); - - Transfer transfer4 = new Transfer(); - transfer4.operation = TransferOperation.MOVE; - transfer4.fromKey = "move.key.4"; - transfer4.toKey = "move.to.key.4"; - transfer4.ifKey = "no.such.key"; - transfer4.ifType = "zoo.cfg"; - transfer4.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer4); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(5, map.size()); - String[] shouldNotExitKeys = new String[]{"move.key.1", "move.key.2", "move.key.3", "move.key.4"}; - String[] shouldExitKeys = new String[]{"move.to.key.1", "move.to.key.2", "move.to.key.3", "move.to.key.4"}; - for(String key: shouldNotExitKeys){ - assertFalse(map.containsKey(key)); - } - - for(String key: shouldExitKeys){ - assertTrue(map.containsKey(key)); - } - } - - @Test - public void testDisallowedTransferMove() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("move.key.1", "m1"); - put("move.key.2", "m2"); - put("move.key.3", "m3"); - put("move.key.4", "m4"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - List transfers = new ArrayList<>(); - Transfer transfer2 = new Transfer(); - transfer2.operation = TransferOperation.MOVE; - transfer2.fromKey = "move.key.2"; - transfer2.toKey = "move.to.key.2"; - transfer2.ifKey = "initLimit"; - transfer2.ifType = "zoo.cfg"; - transfer2.ifValue = "not-real-value"; - transfers.add(transfer2); - - Transfer transfer3 = new Transfer(); - transfer3.operation = TransferOperation.MOVE; - transfer3.fromKey = "move.key.3"; - transfer3.toKey = "move.to.key.3"; - transfer3.ifKey = "initLimit"; - transfer3.ifType = "zoo.cfg"; - transfer3.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer3); - - Transfer transfer4 = new Transfer(); - transfer4.operation = TransferOperation.MOVE; - transfer4.fromKey = "move.key.4"; - transfer4.toKey = "move.to.key.4"; - transfer4.ifKey = "no.such.key"; - transfer4.ifType = "zoo.cfg"; - transfer4.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer3); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(5, map.size()); - - String[] shouldExitKeys = new String[]{"move.key.1", "move.key.2", "move.key.3", "move.key.4"}; - String[] shouldNotExitKeys = new String[]{"move.to.key.1", "move.to.key.2", "move.to.key.3", "move.to.key.4"}; - for(String key: shouldNotExitKeys){ - assertFalse(map.containsKey(key)); - } - - for(String key: shouldExitKeys){ - assertTrue(map.containsKey(key)); - } - } - - @Test - public void testAllowedTransferDelete() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("delete.key.1", "d1"); - put("delete.key.2", "d2"); - put("delete.key.3", "d3"); - put("delete.key.4", "d4"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - List transfers = new ArrayList<>(); - Transfer transfer1 = new Transfer(); - transfer1.operation = TransferOperation.DELETE; - transfer1.deleteKey = "delete.key.1"; - transfers.add(transfer1); - - Transfer transfer2 = new Transfer(); - transfer2.operation = TransferOperation.DELETE; - transfer2.deleteKey = "delete.key.2"; - transfer2.ifKey = "initLimit"; - transfer2.ifType = "zoo.cfg"; - transfer2.ifValue = "10"; - transfers.add(transfer2); - - Transfer transfer3 = new Transfer(); - transfer3.operation = TransferOperation.DELETE; - transfer3.deleteKey = "delete.key.3"; - transfer3.ifKey = "initLimit"; - transfer3.ifType = "zoo.cfg"; - transfer3.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer3); - - Transfer transfer4 = new Transfer(); - transfer4.operation = TransferOperation.DELETE; - transfer4.deleteKey = "delete.key.4"; - transfer4.ifKey = "no.such.key"; - transfer4.ifType = "zoo.cfg"; - transfer4.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer4); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(1, map.size()); - assertEquals("11", map.get("initLimit")); - String[] shouldNotExitKeys = new String[]{"delete.key.1","delete.key.2","delete.key.3","delete.key.4"}; - for(String key: shouldNotExitKeys){ - assertFalse(map.containsKey(key)); - } - } - - @Test - public void testDisallowedTransferDelete() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - put("delete.key.1", "d1"); - put("delete.key.2", "d2"); - put("delete.key.3", "d3"); - put("delete.key.4", "d4"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - List configurations = new ArrayList<>(); - ConfigurationKeyValue keyValue = new ConfigurationKeyValue(); - configurations.add(keyValue); - keyValue.key = "initLimit"; - keyValue.value = "11"; - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - commandParams.put(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS, new Gson().toJson(configurations)); - - List transfers = new ArrayList<>(); - Transfer transfer2 = new Transfer(); - transfer2.operation = TransferOperation.DELETE; - transfer2.deleteKey = "delete.key.2"; - transfer2.ifKey = "initLimit"; - transfer2.ifType = "zoo.cfg"; - transfer2.ifValue = "not.real.value"; - transfers.add(transfer2); - - Transfer transfer3 = new Transfer(); - transfer3.operation = TransferOperation.DELETE; - transfer3.deleteKey = "delete.key.3"; - transfer3.ifKey = "initLimit"; - transfer3.ifType = "zoo.cfg"; - transfer3.ifKeyState = PropertyKeyState.ABSENT; - transfers.add(transfer3); - - Transfer transfer4 = new Transfer(); - transfer4.operation = TransferOperation.DELETE; - transfer4.deleteKey = "delete.key.4"; - transfer4.ifKey = "no.such.key"; - transfer4.ifType = "zoo.cfg"; - transfer4.ifKeyState = PropertyKeyState.PRESENT; - transfers.add(transfer4); - - commandParams.put(ConfigureTask.PARAMETER_TRANSFERS, new Gson().toJson(transfers)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, - null, null); - - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper( - executionCommand)); - - - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - Map map = config.getProperties(); - assertEquals(5, map.size()); - assertEquals("11", map.get("initLimit")); - String[] shouldExitKeys = new String[]{"delete.key.1","delete.key.2","delete.key.3","delete.key.4"}; - for(String key: shouldExitKeys){ - assertTrue(map.containsKey(key)); - } - } - - /** - * Tests using the {@code } element in a configuration upgrade pack. - * - * @throws Exception - */ - @Test - public void testInsert() throws Exception { - Cluster c = clusters.getCluster("c1"); - assertEquals(1, c.getConfigsByType("zoo.cfg").size()); - - Map properties = new HashMap() { - { - put("key_to_append", "append"); - put("key_to_prepend", "prepend"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version2", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - assertEquals(2, c.getConfigsByType("zoo.cfg").size()); - - createUpgrade(c, repoVersion2111); - - Map commandParams = new HashMap<>(); - commandParams.put("clusterName", "c1"); - commandParams.put(ConfigureTask.PARAMETER_CONFIG_TYPE, "zoo.cfg"); - - // define the changes - final String prependValue = "This should be on a newline"; - final String appendValue = " this will be after..."; - - // insert tasks - List insertions = new ArrayList<>(); - - Insert prepend = new Insert(); - prepend.insertType = InsertType.PREPEND; - prepend.key = "key_to_prepend"; - prepend.value = prependValue; - prepend.newlineBefore = false; - prepend.newlineAfter = true; - - Insert append = new Insert(); - append.insertType = InsertType.APPEND; - append.key = "key_to_append"; - append.value = appendValue; - append.newlineBefore = false; - append.newlineAfter = false; - - // add them to the list - insertions.add(prepend); - insertions.add(append); - - // just for fun, add them again - this will test their idempotence - insertions.add(prepend); - insertions.add(append); - - commandParams.put(ConfigureTask.PARAMETER_INSERTIONS, new Gson().toJson(insertions)); - - ExecutionCommand executionCommand = getExecutionCommand(commandParams); - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(3, c.getConfigsByType("zoo.cfg").size()); - - config = c.getDesiredConfigByType("zoo.cfg"); - assertNotNull(config); - assertFalse("version2".equals(config.getTag())); - - // build the expected values - String expectedPrepend = prependValue + System.lineSeparator() + "prepend"; - String expectedAppend = "append" + appendValue; - - assertEquals(expectedPrepend, config.getProperties().get("key_to_prepend")); - assertEquals(expectedAppend, config.getProperties().get("key_to_append")); - } - - /** - * Creates a cluster using {@link #repoVersion2110} with ZooKeeper installed. - */ - private void makeUpgradeCluster() throws Exception { - String clusterName = "c1"; - String hostName = "h1"; - - clusters.addCluster(clusterName, repoVersion2110.getStackId()); - - Cluster c = clusters.getCluster(clusterName); - - // add a host component - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - - // !!! very important, otherwise the loops that walk the list of installed - // service properties will not run! - ServiceGroup serviceGroup = c.addServiceGroup("CORE"); - Service service = installService(c, serviceGroup, "ZOOKEEPER", repoVersion2110); - addServiceComponent(c, service, "ZOOKEEPER_SERVER"); - addServiceComponent(c, service, "ZOOKEEPER_CLIENT"); - createNewServiceComponentHost(c, "ZOOKEEPER", "ZOOKEEPER_SERVER", hostName); - createNewServiceComponentHost(c, "ZOOKEEPER", "ZOOKEEPER_CLIENT", hostName); - - Map properties = new HashMap() { - { - put("initLimit", "10"); - } - }; - - Config config = createConfig(c, repoVersion2110, "zoo.cfg", "version1", properties); - - c.addDesiredConfig("user", Collections.singleton(config)); - - // verify that our configs are there - String tickTime = m_configHelper.getPropertyValueFromStackDefinitions(c, "zoo.cfg", "tickTime"); - assertNotNull(tickTime); - } - - /** - * Installs a service in the cluster. - * - * @param cluster - * @param serviceGroup - * @param serviceName - * @return - * @throws AmbariException - */ - private Service installService(Cluster cluster, ServiceGroup serviceGroup, String serviceName, - RepositoryVersionEntity repositoryVersion - ) throws AmbariException { - Service service; - - try { - service = cluster.getService(serviceName); - } catch (ServiceNotFoundException e) { - service = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), serviceName, serviceName, repositoryVersion); - cluster.addService(service); - } - - return service; - } - - private ServiceComponent addServiceComponent(Cluster cluster, Service service, - String componentName) throws AmbariException { - ServiceComponent serviceComponent = null; - try { - serviceComponent = service.getServiceComponent(componentName); - } catch (ServiceComponentNotFoundException e) { - serviceComponent = serviceComponentFactory.createNew(service, componentName); - service.addServiceComponent(serviceComponent); - serviceComponent.setDesiredState(State.INSTALLED); - } - - return serviceComponent; - } - - private ServiceComponentHost createNewServiceComponentHost(Cluster cluster, String serviceName, - String svcComponent, String hostName) throws AmbariException { - Assert.assertNotNull(cluster.getConfigGroups()); - Service s = cluster.getService(serviceName); - ServiceComponent sc = addServiceComponent(cluster, s, svcComponent); - - ServiceComponentHost sch = serviceComponentHostFactory.createNew(sc, hostName); - - sc.addServiceComponentHost(sch); - sch.setDesiredState(State.INSTALLED); - sch.setState(State.INSTALLED); - return sch; - } - - /** - * Creates an upgrade and associates it with the cluster. - */ - private UpgradeEntity createUpgrade(Cluster cluster, RepositoryVersionEntity repositoryVersion) - throws Exception { - - // create some entities for the finalize action to work with for patch - // history - RequestEntity requestEntity = new RequestEntity(); - requestEntity.setClusterId(cluster.getClusterId()); - requestEntity.setRequestId(1L); - requestEntity.setStartTime(System.currentTimeMillis()); - requestEntity.setCreateTime(System.currentTimeMillis()); - requestDAO.create(requestEntity); - - UpgradeEntity upgradeEntity = new UpgradeEntity(); - upgradeEntity.setId(1L); - upgradeEntity.setClusterId(cluster.getClusterId()); - upgradeEntity.setRequestEntity(requestEntity); - upgradeEntity.setUpgradePackage(""); - upgradeEntity.setRepositoryVersion(repositoryVersion); - upgradeEntity.setUpgradeType(UpgradeType.NON_ROLLING); - - Map services = cluster.getServices(); - for (String serviceName : services.keySet()) { - Service service = services.get(serviceName); - Map components = service.getServiceComponents(); - for (String componentName : components.keySet()) { - UpgradeHistoryEntity history = new UpgradeHistoryEntity(); - history.setUpgrade(upgradeEntity); - history.setServiceName(serviceName); - history.setComponentName(componentName); - history.setFromRepositoryVersion(service.getDesiredRepositoryVersion()); - history.setTargetRepositoryVersion(repositoryVersion); - upgradeEntity.addHistory(history); - } - } - - upgradeDAO.create(upgradeEntity); - cluster.setUpgradeEntity(upgradeEntity); - return upgradeEntity; - } - - private ExecutionCommand getExecutionCommand(Map commandParams) { - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setClusterName("c1"); - executionCommand.setCommandParams(commandParams); - executionCommand.setRoleParams(new HashMap<>()); - executionCommand.getRoleParams().put(ServerAction.ACTION_USER_NAME, "username"); - - return executionCommand; - } - - private Config createConfig(Cluster cluster, RepositoryVersionEntity repoVersion, String type, - String tag, Map properties) { - return configFactory.createNew(repoVersion.getStackId(), cluster, type, tag, properties, - NO_ATTRIBUTES, 1L); - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/CreateAndConfigureActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/CreateAndConfigureActionTest.java index dfc14046dba..21c43b4ac68 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/CreateAndConfigureActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/CreateAndConfigureActionTest.java @@ -222,7 +222,7 @@ private void makeUpgradeCluster() throws Exception { // !!! very important, otherwise the loops that walk the list of installed // service properties will not run! - ServiceGroup serviceGroup = c.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c.addServiceGroup("CORE", repoVersion2110.getStackId().getStackId()); Service zk = installService(c, serviceGroup, "ZOOKEEPER", repoVersion2110); addServiceComponent(zk, "ZOOKEEPER_SERVER"); addServiceComponent(zk, "ZOOKEEPER_CLIENT"); @@ -312,7 +312,7 @@ private UpgradeEntity createUpgrade(Cluster cluster, RepositoryVersionEntity rep upgradeEntity.setRequestEntity(requestEntity); upgradeEntity.setUpgradePackage(""); upgradeEntity.setRepositoryVersion(repositoryVersion); - upgradeEntity.setUpgradeType(UpgradeType.NON_ROLLING); + upgradeEntity.setUpgradeType(UpgradeType.EXPRESS); Map services = cluster.getServices(); for (String serviceName : services.keySet()) { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java deleted file mode 100644 index d26e22b85ec..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ /dev/null @@ -1,696 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.serveraction.upgrades; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.lang.reflect.Field; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import javax.persistence.EntityManager; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.ServiceComponentNotFoundException; -import org.apache.ambari.server.ServiceNotFoundException; -import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; -import org.apache.ambari.server.actionmanager.HostRoleCommand; -import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; -import org.apache.ambari.server.actionmanager.HostRoleStatus; -import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; -import org.apache.ambari.server.controller.ServiceConfigVersionResponse; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.OrmTestHelper; -import org.apache.ambari.server.orm.dao.HostComponentStateDAO; -import org.apache.ambari.server.orm.dao.HostDAO; -import org.apache.ambari.server.orm.dao.HostVersionDAO; -import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; -import org.apache.ambari.server.orm.dao.RequestDAO; -import org.apache.ambari.server.orm.dao.StackDAO; -import org.apache.ambari.server.orm.dao.UpgradeDAO; -import org.apache.ambari.server.orm.entities.HostComponentStateEntity; -import org.apache.ambari.server.orm.entities.HostVersionEntity; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.RequestEntity; -import org.apache.ambari.server.orm.entities.StackEntity; -import org.apache.ambari.server.orm.entities.UpgradeEntity; -import org.apache.ambari.server.orm.entities.UpgradeHistoryEntity; -import org.apache.ambari.server.serveraction.ServerAction; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.ambari.server.state.Config; -import org.apache.ambari.server.state.ConfigFactory; -import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.RepositoryVersionState; -import org.apache.ambari.server.state.Service; -import org.apache.ambari.server.state.ServiceComponent; -import org.apache.ambari.server.state.ServiceComponentFactory; -import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceComponentHostFactory; -import org.apache.ambari.server.state.ServiceFactory; -import org.apache.ambari.server.state.ServiceGroup; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.State; -import org.apache.ambari.server.state.UpgradeState; -import org.apache.ambari.server.state.stack.UpgradePack; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; -import org.apache.ambari.server.utils.EventBusSynchronizer; -import org.apache.commons.lang.StringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.persist.UnitOfWork; - -/** - * Tests upgrade-related server side actions - */ -public class UpgradeActionTest { - private static final String clusterName = "c1"; - - private static final String HDP_2_1_1_0 = "2.1.1.0-1"; - private static final String HDP_2_1_1_1 = "2.1.1.1-2"; - - private static final String HDP_2_2_0_1 = "2.2.0.1-3"; - - private static final StackId HDP_21_STACK = new StackId("HDP-2.1.1"); - private static final StackId HDP_22_STACK = new StackId("HDP-2.2.0"); - - private RepositoryVersionEntity sourceRepositoryVersion; - - private Injector m_injector; - - private AmbariManagementController amc; - @Inject - private OrmTestHelper m_helper; - @Inject - private RepositoryVersionDAO repoVersionDAO; - @Inject - private Clusters clusters; - @Inject - private HostVersionDAO hostVersionDAO; - @Inject - private HostDAO hostDAO; - @Inject - private HostRoleCommandFactory hostRoleCommandFactory; - @Inject - private ServiceFactory serviceFactory; - @Inject - private ServiceComponentFactory serviceComponentFactory; - @Inject - private ServiceComponentHostFactory serviceComponentHostFactory; - @Inject - private RequestDAO requestDAO; - @Inject - private UpgradeDAO upgradeDAO; - @Inject - private StackDAO stackDAO; - @Inject - private AmbariMetaInfo ambariMetaInfo; - @Inject - private FinalizeUpgradeAction finalizeUpgradeAction; - @Inject - private ConfigFactory configFactory; - - @Inject - private HostComponentStateDAO hostComponentStateDAO; - - private RepositoryVersionEntity repositoryVersion2110; - private RepositoryVersionEntity repositoryVersion2111; - private RepositoryVersionEntity repositoryVersion2201; - - @Before - public void setup() throws Exception { - m_injector = Guice.createInjector(new InMemoryDefaultTestModule()); - EventBusSynchronizer.synchronizeAmbariEventPublisher(m_injector); - m_injector.getInstance(GuiceJpaInitializer.class); - m_injector.injectMembers(this); - m_injector.getInstance(UnitOfWork.class).begin(); - - // Initialize AmbariManagementController - amc = m_injector.getInstance(AmbariManagementController.class); - - Field field = AmbariServer.class.getDeclaredField("clusterController"); - field.setAccessible(true); - field.set(null, amc); - - repositoryVersion2110 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_0); - repositoryVersion2111 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_1); - repositoryVersion2201 = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_0_1); - } - - @After - public void teardown() throws Exception { - m_injector.getInstance(UnitOfWork.class).end(); - H2DatabaseCleaner.clearDatabase(m_injector.getProvider(EntityManager.class).get()); - } - - private void makeDowngradeCluster(RepositoryVersionEntity sourceRepoVersion, - RepositoryVersionEntity targetRepoVersion) throws Exception { - String hostName = "h1"; - - clusters.addCluster(clusterName, sourceRepoVersion.getStackId()); - - // add a host component - clusters.addHost(hostName); - - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - host.setHostAttributes(hostAttributes); - - HostVersionEntity entity = new HostVersionEntity(); - entity.setHostEntity(hostDAO.findByName(hostName)); - entity.setRepositoryVersion(targetRepoVersion); - entity.setState(RepositoryVersionState.INSTALLING); - hostVersionDAO.create(entity); - } - - private Cluster createUpgradeCluster( - RepositoryVersionEntity sourceRepoVersion, String hostName) throws Exception { - - clusters.addCluster(clusterName, sourceRepoVersion.getStackId()); - Cluster cluster = clusters.getCluster(clusterName); - - // add a host component - clusters.addHost(hostName); - - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - host.setHostAttributes(hostAttributes); - - // without this, HostEntity will not have a relation to ClusterEntity - clusters.mapHostToCluster(hostName, clusterName); - - HostVersionEntity entity = new HostVersionEntity(hostDAO.findByName(hostName), - sourceRepoVersion, RepositoryVersionState.INSTALLED); - - hostVersionDAO.create(entity); - - return cluster; - } - - private void createHostVersions(RepositoryVersionEntity targetRepoVersion, - String hostName) throws AmbariException { - Cluster c = clusters.getCluster(clusterName); - - // create a single host with the UPGRADED HostVersionEntity - HostDAO hostDAO = m_injector.getInstance(HostDAO.class); - - HostVersionEntity entity = new HostVersionEntity(hostDAO.findByName(hostName), - targetRepoVersion, RepositoryVersionState.INSTALLED); - - hostVersionDAO.create(entity); - - // verify the UPGRADED host versions were created successfully - List hostVersions = hostVersionDAO.findHostVersionByClusterAndRepository( - c.getClusterId(), targetRepoVersion); - - assertEquals(1, hostVersions.size()); - assertEquals(RepositoryVersionState.INSTALLED, hostVersions.get(0).getState()); - } - - private void makeCrossStackUpgradeClusterAndSourceRepo(StackId sourceStack, String sourceRepo, - String hostName)throws Exception { - - clusters.addCluster(clusterName, sourceStack); - - StackEntity stackEntitySource = stackDAO.find(sourceStack.getStackName(), sourceStack.getStackVersion()); - - assertNotNull(stackEntitySource); - - Cluster c = clusters.getCluster(clusterName); - c.setCurrentStackVersion(sourceStack); - c.setDesiredStackVersion(sourceStack); - - // add a host component - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - - // Create the starting repo version - sourceRepositoryVersion = m_helper.getOrCreateRepositoryVersion(sourceStack, sourceRepo); - } - - private void makeCrossStackUpgradeTargetRepo(StackId targetStack, String targetRepo, String hostName) throws Exception{ - StackEntity stackEntityTarget = stackDAO.find(targetStack.getStackName(), targetStack.getStackVersion()); - assertNotNull(stackEntityTarget); - - m_helper.getOrCreateRepositoryVersion(new StackId(stackEntityTarget), targetRepo); - - // Start upgrading the newer repo - - HostDAO hostDAO = m_injector.getInstance(HostDAO.class); - - HostVersionEntity entity = new HostVersionEntity(); - entity.setHostEntity(hostDAO.findByName(hostName)); - entity.setRepositoryVersion(repoVersionDAO.findByStackAndVersion(targetStack, targetRepo)); - entity.setState(RepositoryVersionState.INSTALLED); - hostVersionDAO.create(entity); - } - - /*** - * During an Express Upgrade that crosses a stack version, Ambari calls UpdateDesiredRepositoryAction - * in order to change the stack and apply configs. - * The configs that are applied must be saved with the username that is passed in the role params. - */ - @Test - public void testExpressUpgradeUpdateDesiredRepositoryAction() throws Exception { - StackId sourceStack = HDP_21_STACK; - StackId targetStack = HDP_22_STACK; - String sourceRepo = HDP_2_1_1_0; - String hostName = "h1"; - - // Must be a NON_ROLLING upgrade that jumps stacks in order for it to apply config changes. - // That upgrade pack has changes for ZK and NameNode. - String upgradePackName = "upgrade_nonrolling_new_stack"; - - Map packs = ambariMetaInfo.getUpgradePacks(sourceStack.getStackName(), sourceStack.getStackVersion()); - Assert.assertTrue(packs.containsKey(upgradePackName)); - - makeCrossStackUpgradeClusterAndSourceRepo(sourceStack, sourceRepo, hostName); - - Cluster cluster = clusters.getCluster(clusterName); - - // Install ZK and HDFS with some components - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); - Service zk = installService(cluster, serviceGroup, "ZOOKEEPER", repositoryVersion2110); - addServiceComponent(zk, "ZOOKEEPER_SERVER"); - addServiceComponent(zk, "ZOOKEEPER_CLIENT"); - createNewServiceComponentHost(cluster, serviceGroup, "ZOOKEEPER", "ZOOKEEPER_SERVER", "h1"); - createNewServiceComponentHost(cluster, serviceGroup, "ZOOKEEPER", "ZOOKEEPER_CLIENT", "h1"); - - Service hdfs = installService(cluster, serviceGroup, "HDFS", repositoryVersion2110); - addServiceComponent(hdfs, "NAMENODE"); - addServiceComponent(hdfs, "DATANODE"); - createNewServiceComponentHost(cluster, serviceGroup, "HDFS", "NAMENODE", "h1"); - createNewServiceComponentHost(cluster, serviceGroup, "HDFS", "DATANODE", "h1"); - - makeCrossStackUpgradeTargetRepo(targetStack, repositoryVersion2201.getVersion(), hostName); - createUpgrade(cluster, repositoryVersion2201); - - Assert.assertNotNull(repositoryVersion2201); - - // Create some configs - createConfigs(cluster); - Collection configs = cluster.getAllConfigs(); - Assert.assertFalse(configs.isEmpty()); - - Map commandParams = new HashMap<>(); - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - Map roleParams = new HashMap<>(); - - // User that is performing the config changes - String userName = "admin"; - roleParams.put(ServerAction.ACTION_USER_NAME, userName); - executionCommand.setRoleParams(roleParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - // Call the action to change the desired stack and apply the configs from the Config Pack called by the Upgrade Pack. - UpdateDesiredRepositoryAction action = m_injector.getInstance(UpdateDesiredRepositoryAction.class); - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hostRoleCommand); - - List configVersionsBefore = cluster.getServiceConfigVersions(); - - CommandReport report = action.execute(null); - assertNotNull(report); - - assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - - List configVersionsAfter = cluster.getServiceConfigVersions(); - Assert.assertFalse(configVersionsAfter.isEmpty()); - - assertTrue(configVersionsAfter.size() - configVersionsBefore.size() >= 1); - } - - @Test - public void testFinalizeDowngrade() throws Exception { - makeDowngradeCluster(repositoryVersion2110, repositoryVersion2111); - - Cluster cluster = clusters.getCluster(clusterName); - - createUpgrade(cluster, repositoryVersion2111); - - Map commandParams = new HashMap<>(); - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - finalizeUpgradeAction.setExecutionCommand(executionCommand); - finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand); - - CommandReport report = finalizeUpgradeAction.execute(null); - assertNotNull(report); - assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - - for (HostVersionEntity entity : hostVersionDAO.findByClusterAndHost(clusterName, "h1")) { - if (StringUtils.equals(entity.getRepositoryVersion().getVersion(), repositoryVersion2110.getVersion())) { - assertEquals(RepositoryVersionState.CURRENT, entity.getState()); - } else { - assertEquals(RepositoryVersionState.INSTALLED, entity.getState()); - } - } - } - - @Test - public void testFinalizeUpgrade() throws Exception { - String hostName = "h1"; - - createUpgradeCluster(repositoryVersion2110, hostName); - createHostVersions(repositoryVersion2111, hostName); - - Cluster cluster = clusters.getCluster(clusterName); - - createUpgrade(cluster, repositoryVersion2111); - - // Finalize the upgrade - Map commandParams = new HashMap<>(); - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - finalizeUpgradeAction.setExecutionCommand(executionCommand); - finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand); - - // this should fail since the host versions have not moved to current - CommandReport report = finalizeUpgradeAction.execute(null); - assertEquals(HostRoleStatus.FAILED.name(), report.getStatus()); - - List hostVersions = hostVersionDAO.findHostVersionByClusterAndRepository( - cluster.getClusterId(), repositoryVersion2111); - - for (HostVersionEntity hostVersion : hostVersions) { - hostVersion.setState(RepositoryVersionState.CURRENT); - } - - report = finalizeUpgradeAction.execute(null); - assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - - hostVersions = hostVersionDAO.findHostVersionByClusterAndRepository(cluster.getClusterId(), - repositoryVersion2111); - - for (HostVersionEntity hostVersion : hostVersions) { - Collection hostComponentStates = hostComponentStateDAO.findByHost(hostVersion.getHostName()); - for (HostComponentStateEntity hostComponentStateEntity: hostComponentStates) { - assertEquals(UpgradeState.NONE, hostComponentStateEntity.getUpgradeState()); - } - } - } - - /** - * Tests that finalize still works when there are hosts which are already - * {@link RepositoryVersionState#CURRENT}. - */ - @Test - public void testFinalizeWithHostsAlreadyCurrent() throws Exception { - String hostName = "h1"; - - createUpgradeCluster(repositoryVersion2110, hostName); - createHostVersions(repositoryVersion2111, hostName); - - // move the old version from CURRENT to INSTALLED and the new version from - // UPGRADED to CURRENT - this will simulate what happens when a host is - // removed before finalization and all hosts transition to CURRENT - List hostVersions = hostVersionDAO.findAll(); - for (HostVersionEntity hostVersion : hostVersions) { - if (hostVersion.getState() == RepositoryVersionState.CURRENT) { - hostVersion.setState(RepositoryVersionState.INSTALLED); - } else { - hostVersion.setState(RepositoryVersionState.CURRENT); - } - - hostVersionDAO.merge(hostVersion); - } - - // Verify the repo before calling Finalize - Cluster cluster = clusters.getCluster(clusterName); - - createUpgrade(cluster, repositoryVersion2111); - - // Finalize the upgrade - Map commandParams = new HashMap<>(); - - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - finalizeUpgradeAction.setExecutionCommand(executionCommand); - finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand); - - CommandReport report = finalizeUpgradeAction.execute(null); - assertNotNull(report); - assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - } - - /** - * Tests that all host versions are correct after upgrade. This test will - * ensure that the prior CURRENT versions are moved to INSTALLED while not - * touching any others. - */ - @Test - public void testHostVersionsAfterUpgrade() throws Exception { - String hostName = "h1"; - Cluster cluster = createUpgradeCluster(repositoryVersion2110, hostName); - createHostVersions(repositoryVersion2111, hostName); - createHostVersions(repositoryVersion2201, hostName); - - // Install ZK with some components - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); - Service zk = installService(cluster, serviceGroup, "ZOOKEEPER", repositoryVersion2110); - addServiceComponent(zk, "ZOOKEEPER_SERVER"); - addServiceComponent(zk, "ZOOKEEPER_CLIENT"); - createNewServiceComponentHost(cluster, serviceGroup, "ZOOKEEPER", "ZOOKEEPER_SERVER", hostName); - createNewServiceComponentHost(cluster, serviceGroup, "ZOOKEEPER", "ZOOKEEPER_CLIENT", hostName); - - List hostVersions = hostVersionDAO.findAll(); - assertEquals(3, hostVersions.size()); - - // repo 2110 - CURRENT (upgrading from) - // repo 2111 - CURRENT (all hosts reported in during upgrade) - // repo 2201 - NOT_REQUIRED (different stack) - for (HostVersionEntity hostVersion : hostVersions) { - RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion(); - if (repositoryVersion2110.equals(hostRepoVersion)) { - hostVersion.setState(RepositoryVersionState.CURRENT); - } else if (repositoryVersion2111.equals(hostRepoVersion)) { - hostVersion.setState(RepositoryVersionState.CURRENT); - } else { - hostVersion.setState(RepositoryVersionState.NOT_REQUIRED); - } - - hostVersionDAO.merge(hostVersion); - } - - // upgrade to 2111 - createUpgrade(cluster, repositoryVersion2111); - - // push all services to the correct repo version for finalize - Map services = cluster.getServices(); - assertTrue(services.size() > 0); - for (Service service : services.values()) { - service.setDesiredRepositoryVersion(repositoryVersion2111); - } - - // push all components to the correct version - List hostComponentStates = hostComponentStateDAO.findByHost(hostName); - for (HostComponentStateEntity hostComponentState : hostComponentStates) { - hostComponentState.setVersion(repositoryVersion2111.getVersion()); - hostComponentStateDAO.merge(hostComponentState); - } - - Map commandParams = new HashMap<>(); - ExecutionCommand executionCommand = new ExecutionCommand(); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName(clusterName); - - HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); - hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); - - finalizeUpgradeAction.setExecutionCommand(executionCommand); - finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand); - - // finalize - CommandReport report = finalizeUpgradeAction.execute(null); - assertNotNull(report); - assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - - for (HostVersionEntity hostVersion : hostVersions) { - RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion(); - if (repositoryVersion2110.equals(hostRepoVersion)) { - assertEquals(RepositoryVersionState.INSTALLED, hostVersion.getState()); - } else if (repositoryVersion2111.equals(hostRepoVersion)) { - assertEquals(RepositoryVersionState.CURRENT, hostVersion.getState()); - } else { - assertEquals(RepositoryVersionState.NOT_REQUIRED, hostVersion.getState()); - } - } - } - - private ServiceComponentHost createNewServiceComponentHost(Cluster cluster, ServiceGroup serviceGroup, String svc, - String svcComponent, String hostName - ) throws AmbariException { - Assert.assertNotNull(cluster.getConfigGroups()); - Service s = installService(cluster, serviceGroup, svc, sourceRepositoryVersion); - ServiceComponent sc = addServiceComponent(s, svcComponent); - - ServiceComponentHost sch = serviceComponentHostFactory.createNew(sc, hostName); - - sc.addServiceComponentHost(sch); - sch.setDesiredState(State.INSTALLED); - sch.setState(State.INSTALLED); - return sch; - } - - private Service installService(Cluster cluster, ServiceGroup serviceGroup, String serviceName, - RepositoryVersionEntity repositoryVersionEntity - ) throws AmbariException { - Service service; - - try { - service = cluster.getService(serviceName); - } catch (ServiceNotFoundException e) { - service = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), serviceName, serviceName, repositoryVersionEntity); - cluster.addService(service); - } - - return service; - } - - private ServiceComponent addServiceComponent(Service service, - String componentName - ) throws AmbariException { - ServiceComponent serviceComponent; - try { - serviceComponent = service.getServiceComponent(componentName); - } catch (ServiceComponentNotFoundException e) { - serviceComponent = serviceComponentFactory.createNew(service, componentName); - service.addServiceComponent(serviceComponent); - serviceComponent.setDesiredState(State.INSTALLED); - } - - return serviceComponent; - } - - private void createConfigs(Cluster cluster) { - Map properties = new HashMap<>(); - Map> propertiesAttributes = new HashMap<>(); - properties.put("a", "a1"); - properties.put("b", "b1"); - - configFactory.createNew(cluster, "zookeeper-env", "version-" + System.currentTimeMillis(), - properties, propertiesAttributes); - - properties.put("zookeeper_a", "value_1"); - properties.put("zookeeper_b", "value_2"); - - configFactory.createNew(cluster, "hdfs-site", "version-" + System.currentTimeMillis(), - properties, propertiesAttributes); - - properties.put("hdfs_a", "value_3"); - properties.put("hdfs_b", "value_4"); - - configFactory.createNew(cluster, "core-site", "version-" + System.currentTimeMillis(), - properties, propertiesAttributes); - - configFactory.createNew(cluster, "foo-site", "version-" + System.currentTimeMillis(), - properties, propertiesAttributes); - } - - /** - * Creates an upgrade and associates it with the cluster. - */ - private UpgradeEntity createUpgrade(Cluster cluster, RepositoryVersionEntity repositoryVersion) - throws Exception { - - // create some entities for the finalize action to work with for patch - // history - RequestEntity requestEntity = new RequestEntity(); - requestEntity.setClusterId(cluster.getClusterId()); - requestEntity.setRequestId(1L); - requestEntity.setStartTime(System.currentTimeMillis()); - requestEntity.setCreateTime(System.currentTimeMillis()); - requestDAO.create(requestEntity); - - UpgradeEntity upgradeEntity = new UpgradeEntity(); - upgradeEntity.setId(1L); - upgradeEntity.setClusterId(cluster.getClusterId()); - upgradeEntity.setRequestEntity(requestEntity); - upgradeEntity.setUpgradePackage(""); - upgradeEntity.setRepositoryVersion(repositoryVersion); - upgradeEntity.setUpgradeType(UpgradeType.NON_ROLLING); - - Map services = cluster.getServices(); - for (String serviceName : services.keySet()) { - Service service = services.get(serviceName); - Map components = service.getServiceComponents(); - for (String componentName : components.keySet()) { - UpgradeHistoryEntity history = new UpgradeHistoryEntity(); - history.setUpgrade(upgradeEntity); - history.setServiceName(serviceName); - history.setComponentName(componentName); - history.setFromRepositoryVersion(service.getDesiredRepositoryVersion()); - history.setTargetRepositoryVersion(repositoryVersion); - upgradeEntity.addHistory(history); - } - } - - upgradeDAO.create(upgradeEntity); - cluster.setUpgradeEntity(upgradeEntity); - return upgradeEntity; - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java index 5e5ef0c99ae..43ea7e1a0cb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java @@ -143,45 +143,4 @@ public void testGetServiceInfoFromSingleStack() throws Exception { verify(metaInfoDao, stackDao, actionMetadata, osFamily); } - /** - * This test ensures that service upgrade xml that creates circular dependencies - * will throw an exception. - */ - @Test - public void testCircularDependencyForServiceUpgrade() throws Exception { - MetainfoDAO metaInfoDao = createNiceMock(MetainfoDAO.class); - StackDAO stackDao = createNiceMock(StackDAO.class); - ExtensionDAO extensionDao = createNiceMock(ExtensionDAO.class); - ExtensionLinkDAO linkDao = createNiceMock(ExtensionLinkDAO.class); - ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class); - OsFamily osFamily = createNiceMock(OsFamily.class); - StackEntity stackEntity = createNiceMock(StackEntity.class); - - expect( - stackDao.find(EasyMock.anyObject(String.class), - EasyMock.anyObject(String.class))).andReturn(stackEntity).atLeastOnce(); - - List list = Collections.emptyList(); - expect( - linkDao.findByStack(EasyMock.anyObject(String.class), - EasyMock.anyObject(String.class))).andReturn(list).atLeastOnce(); - - replay(metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata, osFamily); - - AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); - - try { - String upgradeCycle = ClassLoader.getSystemClassLoader().getResource("stacks_with_upgrade_cycle").getPath(); - File stacksRoot = new File(upgradeCycle); - File resourcesRoot = new File(stacksRoot.getParent()); - - StackManager stackManager = new StackManager(resourcesRoot, stacksRoot, null, null, osFamily, false, false, - metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); - - fail("Expected exception due to cyclic service upgrade xml"); - } catch (AmbariException e) { - // expected - assertEquals("Missing groups: [BAR, FOO]", e.getMessage()); - } - } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index c1cd5fefba1..d71042771f0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -59,10 +59,8 @@ import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.StackInfo; -import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.MetricDefinition; import org.apache.ambari.server.state.stack.OsFamily; -import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.commons.lang.StringUtils; import org.easymock.EasyMock; import org.junit.BeforeClass; @@ -736,29 +734,6 @@ public void testMergeRoleCommandOrder() throws Exception { } - /** - * Tests that {@link UpgradePack} and {@link ConfigUpgradePack} instances are correctly initialized - * post-unmarshalling. - * - * @throws Exception - */ - @Test - public void testUpgradePacksInitializedAfterUnmarshalling() throws Exception { - StackInfo stack = stackManager.getStack("HDP", "2.2.0"); - Map upgradePacks = stack.getUpgradePacks(); - for (UpgradePack upgradePack : upgradePacks.values()) { - assertNotNull(upgradePack); - assertNotNull(upgradePack.getTasks()); - assertTrue(upgradePack.getTasks().size() > 0); - - // reference equality (make sure it's the same list) - assertTrue(upgradePack.getTasks() == upgradePack.getTasks()); - } - ConfigUpgradePack configUpgradePack = stack.getConfigUpgradePack(); - assertNotNull(configUpgradePack); - assertNotNull(configUpgradePack.services); - } - @Test public void testMetricsLoaded() throws Exception { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java index faba98f4cee..dc47bc875fa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java @@ -159,7 +159,7 @@ public void setup() throws Exception { cr2.setType("flume-conf"); cr2.setVersionTag("version1"); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); cluster.addService(serviceGroup, "FLUME", "FLUME", repositoryVersion); cluster.addService(serviceGroup, "OOZIE", "OOZIE", repositoryVersion); cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigMergeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigMergeHelperTest.java deleted file mode 100644 index 8a49e998cc9..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigMergeHelperTest.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.state; - -import static org.apache.ambari.server.state.ConfigMergeHelper.ThreeWayValue; -import static org.easymock.EasyMock.anyString; -import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.sql.SQLException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.util.Modules; - -/** - * Tests the {@link ConfigMergeHelper} class - */ -public class ConfigMergeHelperTest { - - private static final StackId currentStackId = new StackId("HDP-2.1.1"); - private static final StackId newStackId = new StackId("HPD-2.2.0"); - - private Injector injector; - private Clusters clustersMock; - private AmbariMetaInfo ambariMetaInfoMock; - - @Before - public void before() throws Exception { - clustersMock = createNiceMock(Clusters.class); - ambariMetaInfoMock = createNiceMock(AmbariMetaInfo.class); - - final InMemoryDefaultTestModule injectorModule = new InMemoryDefaultTestModule() { - @Override - protected void configure() { - super.configure(); - } - }; - - MockModule mockModule = new MockModule(); - // create an injector which will inject the mocks - injector = Guice.createInjector(Modules.override(injectorModule).with(mockModule)); - injector.getInstance(GuiceJpaInitializer.class); - } - - @After - public void teardown() throws AmbariException, SQLException { - H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); - } - - @Test - public void testGetConflicts() throws Exception { - Cluster clusterMock = createNiceMock(Cluster.class); - - expect(clustersMock.getCluster(anyString())).andReturn(clusterMock); - - expect(clusterMock.getCurrentStackVersion()).andReturn(currentStackId); - expect(clusterMock.getServices()).andReturn(new HashMap() {{ - put("HDFS", createNiceMock(Service.class)); - put("ZK", createNiceMock(Service.class)); - }}); - - - Set currentHDFSProperties = new HashSet() {{ - add(createPropertyInfo("hdfs-env.xml", "equal.key", "equal-value")); - }}; - - Set currentZKProperties = new HashSet() {{ - add(createPropertyInfo("zk-env.xml", "different.key", "different-value-1")); - }}; - - Set currentStackProperties = new HashSet() {{ - add(createPropertyInfo("hadoop-env.xml", "equal.key", "modified.value")); - }}; - - expect(ambariMetaInfoMock.getServiceProperties(currentStackId.getStackName(), - currentStackId.getStackVersion(), "HDFS")).andReturn(currentHDFSProperties); - expect(ambariMetaInfoMock.getServiceProperties(currentStackId.getStackName(), - currentStackId.getStackVersion(), "ZK")).andReturn(currentZKProperties); - expect(ambariMetaInfoMock.getStackProperties(currentStackId.getStackName(), - currentStackId.getStackVersion())).andReturn(currentStackProperties); - - Set newHDFSProperties = new HashSet() {{ - add(createPropertyInfo("hdfs-env.xml", "equal.key", "equal-value")); - add(createPropertyInfo("new-hdfs-config.xml", "equal.key", "equal-value")); - }}; - - Set newZKProperties = new HashSet() {{ - add(createPropertyInfo("zk-env.xml", "equal.key", "different-value-2")); - add(createPropertyInfo("zk-env.xml", "new.key", "new-value-2")); - }}; - - Set newStackProperties = new HashSet() {{ - add(createPropertyInfo("hadoop-env.xml", "equal.key", "another.value")); - }}; - - expect(ambariMetaInfoMock.getServiceProperties(newStackId.getStackName(), - newStackId.getStackVersion(), "HDFS")).andReturn(newHDFSProperties); - expect(ambariMetaInfoMock.getServiceProperties(newStackId.getStackName(), - newStackId.getStackVersion(), "ZK")).andReturn(newZKProperties); - expect(ambariMetaInfoMock.getStackProperties(newStackId.getStackName(), - newStackId.getStackVersion())).andReturn(newStackProperties); - - // desired config of hdfs-env.xml - Map desiredHdfsEnvProperties = new HashMap<>(); - expect(clusterMock.getDesiredConfigByType("hdfs-env.xml")).andReturn( - createConfigMock(desiredHdfsEnvProperties) - ); - - // desired config of zk-env.xml - Map desiredZkEnvProperties = new HashMap<>(); - expect(clusterMock.getDesiredConfigByType("hdfs-env.xml")).andReturn( - createConfigMock(desiredZkEnvProperties) - ); - - // desired config of hadoop-env.xml - Map desiredHadoopEnvProperties = new HashMap<>(); - expect(clusterMock.getDesiredConfigByType("hadoop-env.xml")).andReturn( - createConfigMock(desiredHadoopEnvProperties) - ); - - replay(clusterMock, clustersMock, ambariMetaInfoMock); - - ConfigMergeHelper configMergeHelper = injector.getInstance(ConfigMergeHelper.class); - - Map> conflicts = configMergeHelper.getConflicts( - "clustername", newStackId); - - assertNotNull(conflicts); - assertEquals(2, conflicts.size()); - for (String key : conflicts.keySet()) { - if (key.equals("hdfs-env")) { - Map stringThreeWayValueMap = conflicts.get(key); - assertEquals(1, stringThreeWayValueMap.size()); - assertEquals("equal-value", stringThreeWayValueMap.get("equal.key").oldStackValue); - assertEquals("equal-value", stringThreeWayValueMap.get("equal.key").newStackValue); - assertEquals("", stringThreeWayValueMap.get("equal.key").savedValue); - } else if (key.equals("hadoop-env")) { - Map stringThreeWayValueMap = conflicts.get(key); - assertEquals(1, stringThreeWayValueMap.size()); - assertEquals("modified.value", stringThreeWayValueMap.get("equal.key").oldStackValue); - assertEquals("another.value", stringThreeWayValueMap.get("equal.key").newStackValue); - assertEquals("", stringThreeWayValueMap.get("equal.key").savedValue); - } else { - fail("Unexpected key"); - } - } - assertEquals(2, conflicts.size()); - } - - private PropertyInfo createPropertyInfo(String fileName, String name, String value) { - PropertyInfo result = new PropertyInfo(); - result.setFilename(fileName); - result.setName(name); - result.setValue(value); - return result; - } - - /** - * Generates config that returns properties - * @param properties properties that should be returned by config mock - * @return mock - */ - private Config createConfigMock(Map properties) { - Config result = createNiceMock(Config.class); - expect(result.getProperties()).andReturn(properties); - return result; - } - - @Test - public void testNormalizeValue() throws Exception{ - // If template not defined - String normalizedValue = ConfigMergeHelper.normalizeValue(null, "2048m"); - assertEquals("2048m", normalizedValue); - - // Template does not define heap - normalizedValue = ConfigMergeHelper.normalizeValue("3k", "2048"); - assertEquals("2048", normalizedValue); - - // Template - megabytes - normalizedValue = ConfigMergeHelper.normalizeValue("1024m", "2048"); - assertEquals("2048m", normalizedValue); - - normalizedValue = ConfigMergeHelper.normalizeValue("1024M", "2048"); - assertEquals("2048M", normalizedValue); - - // Template - gigabytes - normalizedValue = ConfigMergeHelper.normalizeValue("4g", "2"); - assertEquals("2g", normalizedValue); - - normalizedValue = ConfigMergeHelper.normalizeValue("4G", "2"); - assertEquals("2G", normalizedValue); - } - - - - private class MockModule implements Module { - - @Override - public void configure(Binder binder) { - binder.bind(Clusters.class).toInstance(clustersMock); - binder.bind(AmbariMetaInfo.class).toInstance(ambariMetaInfoMock); - } - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/MpackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/MpackTest.java index 9184ca58d2d..8523508d465 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/MpackTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/MpackTest.java @@ -30,14 +30,14 @@ public class MpackTest { public void testMpacks() { Mpack mpack = new Mpack(); mpack.setName("name"); - mpack.setMpackId((long)100); + mpack.setResourceId((long)100); mpack.setDescription("desc"); mpack.setVersion("3.0"); mpack.setMpackUri("abc.tar.gz"); mpack.setRegistryId(new Long(100)); Assert.assertEquals("name", mpack.getName()); - Assert.assertEquals(new Long(100), mpack.getMpackId()); + Assert.assertEquals(new Long(100), mpack.getResourceId()); Assert.assertEquals("desc", mpack.getDescription()); Assert.assertEquals("abc.tar.gz", mpack.getMpackUri()); Assert.assertEquals(new Long(100), mpack.getRegistryId()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java index 752c71c5a98..4a4541d6cf7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceComponentTest.java @@ -100,7 +100,7 @@ public void setup() throws Exception { RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service s = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), serviceName, serviceName, repositoryVersion); cluster.addService(s); service = cluster.getService(serviceName); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceTest.java index 6c66c5a075a..b0ee246b373 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ServiceTest.java @@ -90,7 +90,7 @@ public void teardown() throws AmbariException, SQLException { @Test public void testCanBeRemoved() throws Exception{ - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-1.0"); Service service = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); for (State state : State.values()) { @@ -133,7 +133,7 @@ public void testCanBeRemoved() throws Exception{ @Test public void testGetAndSetServiceInfo() throws AmbariException { String serviceName = "HDFS"; - ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", new HashSet()); + ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", STACK_ID, new HashSet()); Service s = serviceFactory.createNew(cluster, serviceGroup, new ArrayList(), serviceName, serviceName, repositoryVersion); cluster.addService(s); @@ -160,7 +160,7 @@ public void testGetAndSetServiceInfo() throws AmbariException { @Test public void testAddGetDeleteServiceComponents() throws AmbariException { String serviceName = "HDFS"; - ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", new HashSet()); + ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", STACK_ID, new HashSet() ); Service s = serviceFactory.createNew(cluster, serviceGroup, new ArrayList(), serviceName, serviceName, repositoryVersion); cluster.addService(s); @@ -243,7 +243,7 @@ public void testGetAndSetConfigs() { @Test public void testConvertToResponse() throws AmbariException { String serviceName = "HDFS"; - ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", new HashSet()); + ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", STACK_ID, new HashSet()); Service s = serviceFactory.createNew(cluster, serviceGroup, new ArrayList(), serviceName, serviceName, repositoryVersion); cluster.addService(s); Service service = cluster.getService(serviceName); @@ -280,7 +280,7 @@ public void testConvertToResponse() throws AmbariException { @Test public void testServiceMaintenance() throws Exception { String serviceName = "HDFS"; - ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", new HashSet()); + ServiceGroup serviceGroup = serviceGroupFactory.createNew(cluster, "service_group", STACK_ID, new HashSet()); Service s = serviceFactory.createNew(cluster, serviceGroup, new ArrayList(), serviceName, serviceName, repositoryVersion); cluster.addService(s); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java index b38b2721fad..6db1d2bb326 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeContextTest.java @@ -236,7 +236,7 @@ public void testPatchUpgrade() throws Exception { replayAll(); Map requestMap = new HashMap<>(); - requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.NON_ROLLING.name()); + requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.EXPRESS.name()); requestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); requestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, m_targetRepositoryVersion.getId().toString()); requestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); @@ -284,7 +284,7 @@ public void testMaintUpgrade() throws Exception { replayAll(); Map requestMap = new HashMap<>(); - requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.NON_ROLLING.name()); + requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.EXPRESS.name()); requestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); requestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, m_targetRepositoryVersion.getId().toString()); requestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); @@ -354,7 +354,7 @@ public void testRevertEU() throws Exception { EasyMock.anyObject(UpgradeType.class), EasyMock.anyString())).andReturn(upgradePack).once(); expect(m_upgradeDAO.findRevertable(1L)).andReturn(m_completedRevertableUpgrade).once(); - expect(m_completedRevertableUpgrade.getUpgradeType()).andReturn(UpgradeType.NON_ROLLING); + expect(m_completedRevertableUpgrade.getUpgradeType()).andReturn(UpgradeType.EXPRESS); Map requestMap = new HashMap<>(); requestMap.put(UpgradeResourceProvider.UPGRADE_REVERT_UPGRADE_ID, "1"); @@ -366,7 +366,7 @@ public void testRevertEU() throws Exception { assertEquals(Direction.DOWNGRADE, context.getDirection()); assertEquals(RepositoryType.PATCH, context.getOrchestrationType()); - assertEquals(UpgradeType.NON_ROLLING, context.getType()); + assertEquals(UpgradeType.EXPRESS, context.getType()); assertEquals(1, context.getSupportedServices().size()); assertTrue(context.isPatchRevert()); @@ -486,7 +486,7 @@ public void testDowngradeForPatch() throws Exception { Map requestMap = new HashMap<>(); - requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.NON_ROLLING.name()); + requestMap.put(UpgradeResourceProvider.UPGRADE_TYPE, UpgradeType.EXPRESS.name()); requestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.DOWNGRADE.name()); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java deleted file mode 100644 index 7c02a6564db..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java +++ /dev/null @@ -1,2911 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.state; - -import static com.google.common.collect.Sets.newLinkedHashSet; -import static java.util.Collections.emptySet; -import static java.util.Collections.singletonList; -import static org.easymock.EasyMock.anyString; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.expectLastCall; -import static org.easymock.EasyMock.replay; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.lang.reflect.Field; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.ambari.annotations.Experimental; -import org.apache.ambari.annotations.ExperimentalFeature; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.api.services.ServiceGroupKey; -import org.apache.ambari.server.api.services.ServiceKey; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.ClusterRequest; -import org.apache.ambari.server.controller.ConfigurationRequest; -import org.apache.ambari.server.controller.internal.UpgradeResourceProvider; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.OrmTestHelper; -import org.apache.ambari.server.orm.dao.ServiceConfigDAO; -import org.apache.ambari.server.orm.entities.ClusterConfigEntity; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.ServiceConfigEntity; -import org.apache.ambari.server.security.TestAuthenticationFactory; -import org.apache.ambari.server.security.authorization.AuthorizationException; -import org.apache.ambari.server.stack.HostsType; -import org.apache.ambari.server.stack.MasterHostResolver; -import org.apache.ambari.server.stack.StackManagerMock; -import org.apache.ambari.server.stageplanner.RoleGraphFactory; -import org.apache.ambari.server.state.UpgradeHelper.UpgradeGroupHolder; -import org.apache.ambari.server.state.stack.ConfigUpgradePack; -import org.apache.ambari.server.state.stack.UpgradePack; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition.ConfigurationKeyValue; -import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; -import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.ExecuteTask; -import org.apache.ambari.server.state.stack.upgrade.Grouping; -import org.apache.ambari.server.state.stack.upgrade.HostOrderGrouping; -import org.apache.ambari.server.state.stack.upgrade.HostOrderItem; -import org.apache.ambari.server.state.stack.upgrade.HostOrderItem.HostOrderActionType; -import org.apache.ambari.server.state.stack.upgrade.ManualTask; -import org.apache.ambari.server.state.stack.upgrade.SecurityCondition; -import org.apache.ambari.server.state.stack.upgrade.ServiceCheckGrouping; -import org.apache.ambari.server.state.stack.upgrade.StageWrapper; -import org.apache.ambari.server.state.stack.upgrade.StopGrouping; -import org.apache.ambari.server.state.stack.upgrade.Task; -import org.apache.ambari.server.state.stack.upgrade.TaskWrapper; -import org.apache.ambari.server.state.stack.upgrade.UpgradeScope; -import org.apache.ambari.server.state.stack.upgrade.UpgradeType; -import org.apache.ambari.server.utils.EventBusSynchronizer; -import org.easymock.Capture; -import org.easymock.EasyMock; -import org.easymock.EasyMockSupport; -import org.easymock.IAnswer; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.springframework.security.core.context.SecurityContextHolder; - -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.Provider; -import com.google.inject.assistedinject.FactoryModuleBuilder; -import com.google.inject.util.Modules; - -/** - * Tests the {@link UpgradeHelper} class - */ -public class UpgradeHelperTest extends EasyMockSupport { - - private static final StackId STACK_ID_HDP_211 = new StackId("HDP-2.1.1"); - private static final StackId STACK_ID_HDP_220 = new StackId("HDP-2.2.0"); - private static final String UPGRADE_VERSION = "2.2.1.0-1234"; - private static final String DOWNGRADE_VERSION = "2.2.0.0-1234"; - - private Injector injector; - private AmbariMetaInfo ambariMetaInfo; - private StackManagerMock stackManagerMock; - private OrmTestHelper helper; - private MasterHostResolver m_masterHostResolver; - private UpgradeHelper m_upgradeHelper; - private ConfigHelper m_configHelper; - private AmbariManagementController m_managementController; - private Gson m_gson = new Gson(); - - private RepositoryVersionEntity repositoryVersion2110; - private RepositoryVersionEntity repositoryVersion2200; - private RepositoryVersionEntity repositoryVersion2210; - private HostsType namenodeHosts = HostsType.highAvailability("h1", "h2", newLinkedHashSet(Arrays.asList("h1", "h2"))); - - /** - * Because test cases need to share config mocks, put common ones in this function. - * @throws Exception - */ - private void setConfigMocks() throws Exception { - // configure the mock to return data given a specific placeholder - m_configHelper = EasyMock.createNiceMock(ConfigHelper.class); - expect(m_configHelper.getPlaceholderValueFromDesiredConfigurations( - EasyMock.anyObject(Cluster.class), eq("{{foo/bar}}"))).andReturn("placeholder-rendered-properly").anyTimes(); - expect(m_configHelper.getEffectiveDesiredTags( - EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).andReturn(new HashMap<>()).anyTimes(); - } - - @Before - public void before() throws Exception { - setConfigMocks(); - // Most test cases can replay the common config mocks. If any test case needs custom ones, it can re-initialize m_configHelper; - replay(m_configHelper); - - final InMemoryDefaultTestModule injectorModule = new InMemoryDefaultTestModule() { - @Override - protected void configure() { - super.configure(); - } - }; - - MockModule mockModule = new MockModule(); - - // create an injector which will inject the mocks - injector = Guice.createInjector(Modules.override(injectorModule).with(mockModule)); - H2DatabaseCleaner.resetSequences(injector); - injector.getInstance(GuiceJpaInitializer.class); - - EventBusSynchronizer.synchronizeAmbariEventPublisher(injector); - EventBusSynchronizer.synchronizeAlertEventPublisher(injector); - - helper = injector.getInstance(OrmTestHelper.class); - ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); - stackManagerMock = (StackManagerMock) ambariMetaInfo.getStackManager(); - m_upgradeHelper = injector.getInstance(UpgradeHelper.class); - m_masterHostResolver = EasyMock.createMock(MasterHostResolver.class); - m_managementController = injector.getInstance(AmbariManagementController.class); - - repositoryVersion2110 = helper.getOrCreateRepositoryVersion(STACK_ID_HDP_211, "2.1.1.0-1234"); - repositoryVersion2200 = helper.getOrCreateRepositoryVersion(STACK_ID_HDP_220, DOWNGRADE_VERSION); - repositoryVersion2210 = helper.getOrCreateRepositoryVersion(STACK_ID_HDP_220, UPGRADE_VERSION); - - // Set the authenticated user - // TODO: remove this or replace the authenticated user to test authorization rules - SecurityContextHolder.getContext().setAuthentication(TestAuthenticationFactory.createAdministrator("admin")); - } - - @After - public void teardown() throws AmbariException, SQLException { - H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); - - // Clear the authenticated user - SecurityContextHolder.getContext().setAuthentication(null); - } - - @Test - public void testSuggestUpgradePack() throws Exception{ - final String clusterName = "c1"; - final StackId sourceStackId = new StackId("HDP", "2.1.1"); - final StackId targetStackId = new StackId("HDP", "2.2.0"); - final Direction upgradeDirection = Direction.UPGRADE; - final UpgradeType upgradeType = UpgradeType.ROLLING; - - makeCluster(); - try { - String preferredUpgradePackName = "upgrade_test"; - UpgradePack up = m_upgradeHelper.suggestUpgradePack(clusterName, sourceStackId, targetStackId, upgradeDirection, upgradeType, preferredUpgradePackName); - assertEquals(upgradeType, up.getType()); - } catch (AmbariException e){ - assertTrue(false); - } - } - - @Test - public void testUpgradeOrchestration() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("ZOOKEEPER", groups.get(1).name); - assertEquals("CORE_MASTER", groups.get(2).name); - assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("HIVE", groups.get(4).name); - assertEquals("OOZIE", groups.get(5).name); - - UpgradeGroupHolder holder = groups.get(2); - boolean found = false; - for (StageWrapper sw : holder.items) { - if (sw.getTasksJson().contains("Upgrading your database")) { - found = true; - } - } - assertTrue("Expected to find replaced text for Upgrading", found); - - UpgradeGroupHolder group = groups.get(1); - // check that the display name is being used - assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); - assertEquals(group.items.get(5).getText(), "Service Check Zk"); - - group = groups.get(3); - assertEquals(8, group.items.size()); - StageWrapper sw = group.items.get(3); - assertEquals("Validate Partial Upgrade", sw.getText()); - assertEquals(1, sw.getTasks().size()); - assertEquals(1, sw.getTasks().get(0).getTasks().size()); - Task t = sw.getTasks().get(0).getTasks().get(0); - assertEquals(ManualTask.class, t.getClass()); - ManualTask mt = (ManualTask) t; - assertTrue(mt.messages.get(0).contains("DataNode and NodeManager")); - assertNotNull(mt.structuredOut); - assertTrue(mt.structuredOut.contains("DATANODE")); - assertTrue(mt.structuredOut.contains("NODEMANAGER")); - - UpgradeGroupHolder postGroup = groups.get(6); - assertEquals("POST_CLUSTER", postGroup.name); - assertEquals("Finalize Upgrade", postGroup.title); - assertEquals(3, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); - assertEquals("Save Cluster State", postGroup.items.get(2).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); - - assertEquals(4, groups.get(0).items.size()); - assertEquals(6, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); - assertEquals(8, groups.get(3).items.size()); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) - @Test - public void testPartialUpgradeOrchestration() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_test_partial")); - UpgradePack upgrade = upgrades.get("upgrade_test_partial"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - Set services = Collections.singleton("ZOOKEEPER"); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING, - repositoryVersion2210, RepositoryType.PATCH, services); - - List groupings = upgrade.getGroups(Direction.UPGRADE); - assertEquals(8, groupings.size()); - assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(3, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("ZOOKEEPER", groups.get(1).name); - assertEquals("POST_CLUSTER", groups.get(2).name); - - UpgradeGroupHolder group = groups.get(1); - // check that the display name is being used - assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); - assertEquals("Service Check Zk", group.items.get(6).getText()); - - UpgradeGroupHolder postGroup = groups.get(2); - assertEquals("POST_CLUSTER", postGroup.name); - assertEquals("Finalize Upgrade", postGroup.title); - assertEquals(2, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Save Cluster State", postGroup.items.get(1).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); - - assertEquals(2, groups.get(0).items.size()); - assertEquals(7, groups.get(1).items.size()); - assertEquals(2, groups.get(2).items.size()); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - @Test - public void testCompleteUpgradeOrchestration() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_test_partial")); - UpgradePack upgrade = upgrades.get("upgrade_test_partial"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING, - repositoryVersion2210, RepositoryType.STANDARD, Collections.singleton("ZOOKEEPER")); - - List groupings = upgrade.getGroups(Direction.UPGRADE); - assertEquals(8, groupings.size()); - assertEquals(UpgradeScope.COMPLETE, groupings.get(6).scope); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(4, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("ZOOKEEPER", groups.get(1).name); - assertEquals("ALL_HOSTS", groups.get(2).name); - assertEquals("POST_CLUSTER", groups.get(3).name); - - UpgradeGroupHolder group = groups.get(1); - // check that the display name is being used - assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); - assertEquals("Service Check Zk", group.items.get(5).getText()); - - UpgradeGroupHolder postGroup = groups.get(3); - assertEquals("POST_CLUSTER", postGroup.name); - assertEquals("Finalize Upgrade", postGroup.title); - assertEquals(2, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Save Cluster State", postGroup.items.get(1).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(1).getType()); - - assertEquals(2, groups.get(0).items.size()); - assertEquals(6, groups.get(1).items.size()); - assertEquals(1, groups.get(2).items.size()); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - @Test - public void testUpgradeServerActionOrchestration() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_server_action_test")); - UpgradePack upgrade = upgrades.get("upgrade_server_action_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(1, groups.size()); - UpgradeGroupHolder group = groups.get(0); - assertEquals("CLUSTER_SERVER_ACTIONS", group.name); - List stageWrappers = group.items; - assertEquals(6, stageWrappers.size()); - assertEquals("Pre Upgrade", stageWrappers.get(0).getText()); - assertEquals("Pre Upgrade Zookeeper", stageWrappers.get(1).getText()); - assertEquals("Configuring", stageWrappers.get(2).getText()); - assertEquals("Configuring HDFS", stageWrappers.get(3).getText()); - assertEquals("Calculating Properties", stageWrappers.get(4).getText()); - assertEquals("Calculating HDFS Properties", stageWrappers.get(5).getText()); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - /** - * Tests that hosts in MM are not included in the upgrade. - * - * @throws Exception - */ - @Test - public void testUpgradeOrchestrationWithHostsInMM() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - // turn on MM for the first host - Cluster cluster = makeCluster(); - Host hostInMaintenanceMode = cluster.getHosts().iterator().next(); - hostInMaintenanceMode.setMaintenanceState(cluster.getClusterId(), MaintenanceState.ON); - - UpgradeContext context = getMockUpgradeContextNoReplay(cluster, Direction.UPGRADE, - UpgradeType.ROLLING, repositoryVersion2210); - - // use a "real" master host resolver here so that we can actually test MM - MasterHostResolver masterHostResolver = new MasterHostResolver(cluster, null, context); - - expect(context.getResolver()).andReturn(masterHostResolver).anyTimes(); - replay(context); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(7, groups.size()); - - for (UpgradeGroupHolder group : groups) { - for (StageWrapper stageWrapper : group.items) { - Set hosts = stageWrapper.getHosts(); - assertFalse(hosts.contains(hostInMaintenanceMode.getHostName())); - } - } - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - /** - * Verify that a Rolling Upgrades restarts the NameNodes in the following order: standby, active. - * @throws Exception - */ - @Test - public void testNamenodeOrder() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - UpgradeGroupHolder mastersGroup = groups.get(2); - assertEquals("CORE_MASTER", mastersGroup.name); - - List orderedNameNodes = new LinkedList<>(); - for (StageWrapper sw : mastersGroup.items) { - if (sw.getType().equals(StageWrapper.Type.RESTART) && sw.getText().toLowerCase().contains("NameNode".toLowerCase())) { - for (TaskWrapper tw : sw.getTasks()) { - for (String hostName : tw.getHosts()) { - orderedNameNodes.add(hostName); - } - } - } - } - - assertEquals(2, orderedNameNodes.size()); - // Order is standby, then active. - assertEquals("h2", orderedNameNodes.get(0)); - assertEquals("h1", orderedNameNodes.get(1)); - } - - @Test - public void testNamenodeFederationOrder() throws Exception { - namenodeHosts = HostsType.federated( - Arrays.asList( - new HostsType.HighAvailabilityHosts("h1", Arrays.asList("h2", "h3")), - new HostsType.HighAvailabilityHosts("h4", singletonList("h5"))), - newLinkedHashSet(Arrays.asList("h1", "h2", "h3", "h4", "h5"))); - - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - UpgradeGroupHolder mastersGroup = groups.get(2); - assertEquals("CORE_MASTER", mastersGroup.name); - - List orderedNameNodes = new LinkedList<>(); - for (StageWrapper sw : mastersGroup.items) { - if (sw.getType().equals(StageWrapper.Type.RESTART) && sw.getText().toLowerCase().contains("NameNode".toLowerCase())) { - for (TaskWrapper tw : sw.getTasks()) { - for (String hostName : tw.getHosts()) { - orderedNameNodes.add(hostName); - } - } - } - } - assertEquals(Arrays.asList("h2", "h3", "h1", "h5", "h4"), orderedNameNodes); - } - - @Test - public void testUpgradeOrchestrationWithNoHeartbeat() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(false); - - Clusters clusters = injector.getInstance(Clusters.class); - Host h4 = clusters.getHost("h4"); - h4.setState(HostState.HEARTBEAT_LOST); - - List schs = cluster.getServiceComponentHosts("h4"); - assertEquals(1, schs.size()); - assertEquals(HostState.HEARTBEAT_LOST, schs.get(0).getHostState()); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("ZOOKEEPER", groups.get(1).name); - assertEquals("CORE_MASTER", groups.get(2).name); - assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("HIVE", groups.get(4).name); - assertEquals("OOZIE", groups.get(5).name); - - UpgradeGroupHolder postGroup = groups.get(6); - assertEquals("POST_CLUSTER", postGroup.name); - assertEquals("Finalize Upgrade", postGroup.title); - assertEquals(3, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); - assertEquals("Save Cluster State", postGroup.items.get(2).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); - - assertEquals(6, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); - assertEquals(7, groups.get(3).items.size()); - } - - @Test - public void testDowngradeOrchestration() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.DOWNGRADE, - UpgradeType.ROLLING, repositoryVersion2200); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("OOZIE", groups.get(1).name); - assertEquals("HIVE", groups.get(2).name); - assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("CORE_MASTER", groups.get(4).name); - assertEquals("ZOOKEEPER", groups.get(5).name); - - - UpgradeGroupHolder postGroup = groups.get(6); - assertEquals("POST_CLUSTER", postGroup.name); - assertEquals("Finalize Downgrade", postGroup.title); - assertEquals(3, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); - assertEquals("Save Cluster State", postGroup.items.get(2).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); - - assertEquals(4, groups.get(0).items.size()); - assertEquals(8, groups.get(1).items.size()); - assertEquals(5, groups.get(2).items.size()); - assertEquals(7, groups.get(3).items.size()); - assertEquals(8, groups.get(4).items.size()); - } - - @Test - public void testBuckets() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_bucket_test")); - UpgradePack upgrade = upgrades.get("upgrade_bucket_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(1, groups.size()); - UpgradeGroupHolder group = groups.iterator().next(); - - // Pre: - // Manual task = 1 - // 2x - Execute task on all 3 = 6 - - // Post: - // Execute task on all 3 = 3 - // 2x - Manual task = 2 - // 3x - Execute task on all 3 = 9 - - // Service Check = 1 - assertEquals(22, group.items.size()); - } - - @Test - public void testManualTaskPostProcessing() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - // grab the manual task out of ZK which has placeholder text - UpgradeGroupHolder zookeeperGroup = groups.get(1); - assertEquals("ZOOKEEPER", zookeeperGroup.name); - ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( - 0).getTasks().get(0); - - assertEquals(1, manualTask.messages.size()); - assertEquals("This is a manual task with a placeholder of placeholder-rendered-properly", - manualTask.messages.get(0)); - } - - @Test - public void testConditionalDeleteTask() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - // grab the configure task out of Hive - UpgradeGroupHolder hiveGroup = groups.get(4); - assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); - - // now change the thrift port to http to have the 2nd condition invoked - Map hiveConfigs = new HashMap<>(); - hiveConfigs.put("hive.server2.transport.mode", "http"); - hiveConfigs.put("hive.server2.thrift.port", "10001"); - hiveConfigs.put("condition", "1"); - - ConfigurationRequest configurationRequest = new ConfigurationRequest(); - configurationRequest.setClusterName(cluster.getClusterName()); - configurationRequest.setType("hive-site"); - configurationRequest.setVersionTag("version2"); - configurationRequest.setProperties(hiveConfigs); - - final ClusterRequest clusterRequest = new ClusterRequest( - cluster.getClusterId(), cluster.getClusterName(), - cluster.getDesiredStackVersion().getStackVersion(), null); - - clusterRequest.setDesiredConfig(singletonList(configurationRequest)); - m_managementController.updateClusters(new HashSet() { - { - add(clusterRequest); - } - }, null); - - Map configProperties = configureTask.getConfigurationChanges(cluster, cup); - assertFalse(configProperties.isEmpty()); - assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); - assertNotNull(configurationJson); - - List transfers = m_gson.fromJson(configurationJson, - new TypeToken>() { }.getType()); - - assertEquals(6, transfers.size()); - assertEquals("copy-key", transfers.get(0).fromKey); - assertEquals("copy-key-to", transfers.get(0).toKey); - - assertEquals("move-key", transfers.get(1).fromKey); - assertEquals("move-key-to", transfers.get(1).toKey); - - assertEquals("delete-key", transfers.get(2).deleteKey); - assertEquals("delete-http-1", transfers.get(3).deleteKey); - assertEquals("delete-http-2", transfers.get(4).deleteKey); - assertEquals("delete-http-3", transfers.get(5).deleteKey); - - - } - - @Test - public void testConfigTaskConditionMet() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, - context); - - assertEquals(7, groups.size()); - - // grab the configure task out of Hive - UpgradeGroupHolder hiveGroup = groups.get(4); - assertEquals("HIVE", hiveGroup.name); - - //Condition is met - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get( - 0).getTasks().get(0); - Map configProperties = configureTask.getConfigurationChanges(cluster, cup); - - assertFalse(configProperties.isEmpty()); - assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS)); - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_REPLACEMENTS)); - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_TRANSFERS)); - - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); - String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); - String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS); - assertNotNull(configurationJson); - assertNotNull(transferJson); - assertNotNull(replacementJson); - - //if conditions for sets... - List keyValuePairs = m_gson.fromJson(configurationJson, - new TypeToken>() { - }.getType()); - assertEquals("setKeyOne", keyValuePairs.get(0).key); - assertEquals("1", keyValuePairs.get(0).value); - - assertEquals("setKeyTwo", keyValuePairs.get(1).key); - assertEquals("2", keyValuePairs.get(1).value); - - assertEquals("setKeyThree", keyValuePairs.get(2).key); - assertEquals("3", keyValuePairs.get(2).value); - - assertEquals("setKeyFour", keyValuePairs.get(3).key); - assertEquals("4", keyValuePairs.get(3).value); - - //if conditions for transfer - List transfers = m_gson.fromJson(transferJson, - new TypeToken>() { - }.getType()); - - assertEquals("copy-key-one", transfers.get(0).fromKey); - assertEquals("copy-to-key-one", transfers.get(0).toKey); - - assertEquals("copy-key-two", transfers.get(1).fromKey); - assertEquals("copy-to-key-two", transfers.get(1).toKey); - - assertEquals("copy-key-three", transfers.get(2).fromKey); - assertEquals("copy-to-key-three", transfers.get(2).toKey); - - assertEquals("copy-key-four", transfers.get(3).fromKey); - assertEquals("copy-to-key-four", transfers.get(3).toKey); - - assertEquals("move-key-one", transfers.get(4).fromKey); - assertEquals("move-to-key-one", transfers.get(4).toKey); - - assertEquals("move-key-two", transfers.get(5).fromKey); - assertEquals("move-to-key-two", transfers.get(5).toKey); - - assertEquals("move-key-three", transfers.get(6).fromKey); - assertEquals("move-to-key-three", transfers.get(6).toKey); - - assertEquals("move-key-four", transfers.get(7).fromKey); - assertEquals("move-to-key-four", transfers.get(7).toKey); - - assertEquals("delete-key-one", transfers.get(8).deleteKey); - assertEquals("delete-key-two", transfers.get(9).deleteKey); - assertEquals("delete-key-three", transfers.get(10).deleteKey); - assertEquals("delete-key-four", transfers.get(11).deleteKey); - - //if conditions for replace - List replacements = m_gson.fromJson(replacementJson, - new TypeToken>() { - }.getType()); - assertEquals("replace-key-one", replacements.get(0).key); - assertEquals("abc", replacements.get(0).find); - assertEquals("abc-replaced", replacements.get(0).replaceWith); - assertEquals("replace-key-two", replacements.get(1).key); - assertEquals("efg", replacements.get(1).find); - assertEquals("efg-replaced", replacements.get(1).replaceWith); - assertEquals("replace-key-three", replacements.get(2).key); - assertEquals("ijk", replacements.get(2).find); - assertEquals("ijk-replaced", replacements.get(2).replaceWith); - assertEquals("replace-key-four", replacements.get(3).key); - assertEquals("lmn", replacements.get(3).find); - assertEquals("lmn-replaced", replacements.get(3).replaceWith); - } - - @Test - public void testConfigTaskConditionSkipped() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, - context); - - assertEquals(7, groups.size()); - - UpgradeGroupHolder hiveGroup = groups.get(4); - assertEquals("HIVE", hiveGroup.name); - - //Condition is not met, so no config operations should be present in the configureTask... - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(3).getTasks().get(0).getTasks().get(0); - Map configProperties = configureTask.getConfigurationChanges(cluster, cup); - - assertFalse(configProperties.isEmpty()); - assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS)); - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_REPLACEMENTS)); - assertTrue(configProperties.containsKey(ConfigureTask.PARAMETER_TRANSFERS)); - - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); - String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); - - String replacementJson = configProperties.get(ConfigureTask.PARAMETER_REPLACEMENTS); - assertNotNull(configurationJson); - assertNotNull(transferJson); - assertNotNull(replacementJson); - - List keyValuePairs = m_gson.fromJson(configurationJson, - new TypeToken>() { - }.getType()); - assertTrue(keyValuePairs.isEmpty()); - - List replacements = m_gson.fromJson(replacementJson, - new TypeToken>() { - }.getType()); - assertTrue(replacements.isEmpty()); - - List transfers = m_gson.fromJson(transferJson, - new TypeToken>() { - }.getType()); - assertTrue(transfers.isEmpty()); - } - - /** - * Tests that {@link ConfigurationKeyValue} pairs on a {@link ConfigureTask} - * are correctly returned based on the if-conditions. - * - * @throws Exception - */ - @Test - public void testConfigureTask() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, - context); - - assertEquals(7, groups.size()); - - // grab the first configure task out of Hive - UpgradeGroupHolder hiveGroup = groups.get(4); - assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); - - Map configProperties = configureTask.getConfigurationChanges(cluster, cup); - assertFalse(configProperties.isEmpty()); - assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - // now set the property in the if-check in the set element so that we have a match - Map hiveConfigs = new HashMap<>(); - hiveConfigs.put("fooKey", "THIS-BETTER-CHANGE"); - hiveConfigs.put("ifFooKey", "ifFooValue"); - - ConfigurationRequest configurationRequest = new ConfigurationRequest(); - configurationRequest.setClusterName(cluster.getClusterName()); - configurationRequest.setType("hive-site"); - configurationRequest.setVersionTag("version2"); - configurationRequest.setProperties(hiveConfigs); - - final ClusterRequest clusterRequest = new ClusterRequest( - cluster.getClusterId(), cluster.getClusterName(), - cluster.getDesiredStackVersion().getStackVersion(), null); - - clusterRequest.setDesiredConfig(singletonList(configurationRequest)); - m_managementController.updateClusters(new HashSet() { - { - add(clusterRequest); - } - }, null); - - // the configure task should now return different properties to set based on - // the if-condition checks - configProperties = configureTask.getConfigurationChanges(cluster, cup); - assertFalse(configProperties.isEmpty()); - assertEquals( configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); - assertNotNull(configurationJson); - - List keyValuePairs = m_gson.fromJson( - configurationJson, - new TypeToken>() { - }.getType()); - - assertEquals("fooKey", keyValuePairs.get(0).key); - assertEquals("fooValue", keyValuePairs.get(0).value); - } - - @Test - public void testConfigureTaskWithMultipleConfigurations() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - assertNotNull(upgrade); - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(7, groups.size()); - - // grab the configure task out of Hive - UpgradeGroupHolder hiveGroup = groups.get(4); - assertEquals("HIVE", hiveGroup.name); - ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(0).getTasks().get(0); - - Map configProperties = configureTask.getConfigurationChanges(cluster, cup); - assertFalse(configProperties.isEmpty()); - assertEquals(configProperties.get(ConfigureTask.PARAMETER_CONFIG_TYPE), "hive-site"); - - String configurationJson = configProperties.get(ConfigureTask.PARAMETER_KEY_VALUE_PAIRS); - String transferJson = configProperties.get(ConfigureTask.PARAMETER_TRANSFERS); - assertNotNull(configurationJson); - assertNotNull(transferJson); - - List keyValuePairs = m_gson.fromJson(configurationJson, - new TypeToken>() { - }.getType()); - - List transfers = m_gson.fromJson(transferJson, - new TypeToken>() { - }.getType()); - - assertEquals("fooKey", keyValuePairs.get(0).key); - assertEquals("fooValue", keyValuePairs.get(0).value); - assertEquals("fooKey2", keyValuePairs.get(1).key); - assertEquals("fooValue2", keyValuePairs.get(1).value); - assertEquals("fooKey3", keyValuePairs.get(2).key); - assertEquals("fooValue3", keyValuePairs.get(2).value); - - assertEquals("copy-key", transfers.get(0).fromKey); - assertEquals("copy-key-to", transfers.get(0).toKey); - - assertEquals("move-key", transfers.get(1).fromKey); - assertEquals("move-key-to", transfers.get(1).toKey); - } - - @Test - public void testServiceCheckUpgradeStages() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - assertNotNull(upgrade); - - // HBASE and PIG have service checks, but not TEZ. - Set additionalServices = new HashSet() {{ add("HBASE"); add("PIG"); add("TEZ"); add("AMBARI_METRICS"); }}; - Cluster c = makeCluster(true, additionalServices); - - int numServiceChecksExpected = 0; - Collection services = c.getServices().values(); - for(Service service : services) { - ServiceInfo si = ambariMetaInfo.getService(c.getCurrentStackVersion().getStackName(), - c.getCurrentStackVersion().getStackVersion(), service.getName()); - if (null == si.getCommandScript()) { - continue; - } - if (service.getName().equalsIgnoreCase("TEZ")) { - assertTrue("Expect Tez to not have any service checks", false); - } - - // Expect AMS to not run any service checks because it is excluded - if (service.getName().equalsIgnoreCase("AMBARI_METRICS")) { - continue; - } - numServiceChecksExpected++; - } - - UpgradeContext context = getMockUpgradeContext(c, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(8, groups.size()); - - UpgradeGroupHolder holder = groups.get(4); - assertEquals(holder.name, "SERVICE_CHECK_1"); - assertEquals(7, holder.items.size()); - int numServiceChecksActual = 0; - for (StageWrapper sw : holder.items) { - for(Service service : services) { - Pattern p = Pattern.compile(".*" + service.getName(), Pattern.CASE_INSENSITIVE); - Matcher matcher = p.matcher(sw.getText()); - if (matcher.matches()) { - numServiceChecksActual++; - continue; - } - } - } - - assertEquals(numServiceChecksActual, numServiceChecksExpected); - - // grab the manual task out of ZK which has placeholder text - UpgradeGroupHolder zookeeperGroup = groups.get(1); - assertEquals("ZOOKEEPER", zookeeperGroup.name); - ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( - 0).getTasks().get(0); - - assertEquals(1, manualTask.messages.size()); - assertEquals( - "This is a manual task with a placeholder of placeholder-rendered-properly", - manualTask.messages.get(0)); - - UpgradeGroupHolder clusterGroup = groups.get(3); - assertEquals(clusterGroup.name, "HBASE"); - assertEquals(clusterGroup.title, "Update HBase Configuration"); - assertEquals(1, clusterGroup.items.size()); - StageWrapper stage = clusterGroup.items.get(0); - assertEquals(stage.getText(), "Update HBase Configuration"); - } - - @Test - public void testServiceCheckDowngradeStages() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.DOWNGRADE, - UpgradeType.ROLLING, repositoryVersion2200); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(6, groups.size()); - - // grab the manual task out of ZK which has placeholder text - - UpgradeGroupHolder zookeeperGroup = groups.get(4); - assertEquals("ZOOKEEPER", zookeeperGroup.name); - ManualTask manualTask = (ManualTask) zookeeperGroup.items.get(0).getTasks().get( - 0).getTasks().get(0); - - assertEquals(1, manualTask.messages.size()); - assertEquals( - "This is a manual task with a placeholder of placeholder-rendered-properly", - manualTask.messages.get(0)); - } - - @Test - public void testUpgradeOrchestrationFullTask() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - assertTrue(upgrades.containsKey("upgrade_to_new_stack")); - UpgradePack upgrade = upgrades.get("upgrade_to_new_stack"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(6, groups.size()); - - assertEquals("PRE_CLUSTER", groups.get(0).name); - assertEquals("ZOOKEEPER", groups.get(1).name); - assertEquals("CORE_MASTER", groups.get(2).name); - assertEquals("CORE_SLAVES", groups.get(3).name); - assertEquals("HIVE", groups.get(4).name); - - UpgradeGroupHolder holder = groups.get(2); - boolean found = false; - for (StageWrapper sw : holder.items) { - if (sw.getTasksJson().contains("Upgrading your database")) { - found = true; - } - } - assertTrue("Expected to find replaced text for Upgrading", found); - - UpgradeGroupHolder group = groups.get(1); - // check that the display name is being used - assertTrue(group.items.get(1).getText().contains("ZooKeeper1 Server2")); - assertEquals(group.items.get(4).getText(), "Service Check Zk"); - - group = groups.get(3); - assertEquals(8, group.items.size()); - StageWrapper sw = group.items.get(3); - assertEquals("Validate Partial Upgrade", sw.getText()); - assertEquals(1, sw.getTasks().size()); - assertEquals(1, sw.getTasks().get(0).getTasks().size()); - Task t = sw.getTasks().get(0).getTasks().get(0); - assertEquals(ManualTask.class, t.getClass()); - ManualTask mt = (ManualTask) t; - assertTrue(mt.messages.get(0).contains("DataNode and NodeManager")); - assertNotNull(mt.structuredOut); - assertTrue(mt.structuredOut.contains("DATANODE")); - assertTrue(mt.structuredOut.contains("NODEMANAGER")); - - UpgradeGroupHolder postGroup = groups.get(5); - assertEquals(postGroup.name, "POST_CLUSTER"); - assertEquals(postGroup.title, "Finalize Upgrade"); - assertEquals(4, postGroup.items.size()); - assertEquals("Confirm Finalize", postGroup.items.get(0).getText()); - assertEquals("Execute HDFS Finalize", postGroup.items.get(1).getText()); - assertEquals("Save Cluster State", postGroup.items.get(2).getText()); - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType()); - assertEquals("Run On All 2.2.1.0-1234", postGroup.items.get(3).getText()); - - assertEquals(1, postGroup.items.get(3).getTasks().size()); - Set hosts = postGroup.items.get(3).getTasks().get(0).getHosts(); - assertNotNull(hosts); - assertEquals(4, hosts.size()); - - assertEquals(4, groups.get(0).items.size()); - assertEquals(5, groups.get(1).items.size()); - assertEquals(9, groups.get(2).items.size()); - assertEquals(8, groups.get(3).items.size()); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - - private Cluster makeCluster() throws AmbariException, AuthorizationException { - return makeCluster(true); - } - - - /** - * Create an HA cluster - * @throws AmbariException - */ - private Cluster makeCluster(boolean clean) throws AmbariException, AuthorizationException { - return makeCluster(clean, new HashSet<>()); - } - - /** - * Create an HA cluster - * @throws AmbariException - */ - private Cluster makeCluster(boolean clean, Set additionalServices) throws AmbariException, AuthorizationException { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - String repositoryVersionString = "2.1.1-1234"; - StackId stackId = new StackId("HDP-2.1.1"); - - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, - repositoryVersionString); - - helper.getOrCreateRepositoryVersion(STACK_ID_HDP_220, "2.2.0"); - - helper.getOrCreateRepositoryVersion(STACK_ID_HDP_220, UPGRADE_VERSION); - - for (int i = 0; i < 4; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // !!! add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HDFS", "HDFS", repositoryVersion)); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "YARN", "YARN", repositoryVersion)); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "ZOOKEEPER", "ZOOKEEPER", repositoryVersion)); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HIVE", "HIVE", repositoryVersion)); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "OOZIE", "OOZIE", repositoryVersion)); - - Service s = c.getService("HDFS"); - ServiceComponent sc = s.addServiceComponent("NAMENODE"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - sc = s.addServiceComponent("DATANODE"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - ServiceComponentHost sch = sc.addServiceComponentHost("h4"); - - s = c.getService("ZOOKEEPER"); - sc = s.addServiceComponent("ZOOKEEPER_SERVER"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - - s = c.getService("YARN"); - sc = s.addServiceComponent("RESOURCEMANAGER"); - sc.addServiceComponentHost("h2"); - - sc = s.addServiceComponent("NODEMANAGER"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h3"); - - s = c.getService("HIVE"); - sc = s.addServiceComponent("HIVE_SERVER"); - sc.addServiceComponentHost("h2"); - - s = c.getService("OOZIE"); - // Oozie Server HA - sc = s.addServiceComponent("OOZIE_SERVER"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - sc = s.addServiceComponent("OOZIE_CLIENT"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - sc.addServiceComponentHost("h3"); - - // set some desired configs - Map hiveConfigs = new HashMap<>(); - hiveConfigs.put("hive.server2.transport.mode", "binary"); - hiveConfigs.put("hive.server2.thrift.port", "10001"); - - ConfigurationRequest configurationRequest = new ConfigurationRequest(); - configurationRequest.setClusterName(clusterName); - configurationRequest.setType("hive-site"); - configurationRequest.setVersionTag("version1"); - configurationRequest.setProperties(hiveConfigs); - - final ClusterRequest clusterRequest = new ClusterRequest(c.getClusterId(), - clusterName, c.getDesiredStackVersion().getStackVersion(), null); - - clusterRequest.setDesiredConfig(singletonList(configurationRequest)); - m_managementController.updateClusters(new HashSet() { - { - add(clusterRequest); - } - }, null); - - HostsType type = HostsType.normal("h1", "h2", "h3"); - expect(m_masterHostResolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER")).andReturn(type).anyTimes(); - expect(m_masterHostResolver.getMasterAndHosts("HDFS", "NAMENODE")).andReturn(namenodeHosts).anyTimes(); - - if (clean) { - type = HostsType.normal("h2", "h3", "h4"); - } else { - type = HostsType.normal("h2", "h3"); - type.unhealthy = singletonList(sch); - } - expect(m_masterHostResolver.getMasterAndHosts("HDFS", "DATANODE")).andReturn(type).anyTimes(); - - type = HostsType.normal("h2"); - expect(m_masterHostResolver.getMasterAndHosts("YARN", "RESOURCEMANAGER")).andReturn(type).anyTimes(); - - type = HostsType.normal(Sets.newLinkedHashSet()); - expect(m_masterHostResolver.getMasterAndHosts("YARN", "APP_TIMELINE_SERVER")).andReturn(type).anyTimes(); - - type = HostsType.normal("h1", "h3"); - expect(m_masterHostResolver.getMasterAndHosts("YARN", "NODEMANAGER")).andReturn(type).anyTimes(); - - expect(m_masterHostResolver.getMasterAndHosts("HIVE", "HIVE_SERVER")).andReturn( - type).anyTimes(); - - type = HostsType.normal("h2", "h3"); - expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_SERVER")).andReturn(type).anyTimes(); - - type = HostsType.normal("h1", "h2", "h3"); - expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_CLIENT")).andReturn(type).anyTimes(); - - expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes(); - - for(String service : additionalServices) { - c.addService(serviceGroup, service, service, repositoryVersion); - if (service.equals("HBASE")) { - type = HostsType.normal("h1", "h2"); - expect(m_masterHostResolver.getMasterAndHosts("HBASE", "HBASE_MASTER")).andReturn(type).anyTimes(); - } - } - - replay(m_masterHostResolver); - - return c; - } - - /** - * Test that multiple execute tasks with an annotation of synchronized="true" each run in their own stage. - */ - @Test - public void testUpgradeWithMultipleTasksInOwnStage() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertNotNull(upgrade); - assertTrue(upgrade.getType() == UpgradeType.ROLLING); - - List upgradePackGroups = upgrade.getGroups(Direction.UPGRADE); - - boolean foundService = false; - for (Grouping group : upgradePackGroups) { - if (group.title.equals("Oozie")) { - foundService = true; - } - } - assertTrue(foundService); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - // The upgrade pack has 2 tasks for Oozie in the pre-upgrade group. - // The first task runs on "all", i.e., both Oozie Servers, whereas the - // second task runs on "any", i.e., exactly one. - int numPrepareStages = 0; - for (UpgradeGroupHolder group : groups) { - if (group.name.equals("OOZIE")) { - assertTrue(group.items.size() > 0); - for (StageWrapper sw : group.items) { - - if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2 (Batch 1 of 2)") || - sw.getText().equalsIgnoreCase("Preparing Oozie Server on h3 (Batch 2 of 2)")) { - numPrepareStages++; - List taskWrappers = sw.getTasks(); - assertEquals(1, taskWrappers.size()); - List tasks = taskWrappers.get(0).getTasks(); - assertEquals(1, taskWrappers.get(0).getHosts().size()); - assertEquals(1, tasks.size()); - - ExecuteTask task = (ExecuteTask) tasks.get(0); - assertTrue("scripts/oozie_server.py".equalsIgnoreCase(task.script)); - assertTrue("stop".equalsIgnoreCase(task.function)); - } - - if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2")) { - numPrepareStages++; - List taskWrappers = sw.getTasks(); - assertEquals(1, taskWrappers.size()); - List tasks = taskWrappers.get(0).getTasks(); - assertEquals(1, taskWrappers.get(0).getHosts().size()); - assertEquals(1, tasks.size()); - - ExecuteTask task = (ExecuteTask) tasks.get(0); - assertTrue("scripts/oozie_server_upgrade.py".equalsIgnoreCase(task.script)); - assertTrue("upgrade_oozie_database_and_sharelib".equalsIgnoreCase(task.function)); - } - } - } - } - assertEquals(3, numPrepareStages); - } - - @Test - public void testDowngradeAfterPartialUpgrade() throws Exception { - - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - String version = "2.1.1.0-1234"; - StackId stackId = new StackId("HDP-2.1.1"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, version); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // !!! add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HDFS", "HDFS", repositoryVersion)); - - Service s = c.getService("HDFS"); - ServiceComponent sc = s.addServiceComponent("NAMENODE"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - - List schs = c.getServiceComponentHosts("HDFS", "NAMENODE"); - assertEquals(2, schs.size()); - - HostsType type = HostsType.highAvailability("h1", "h2", new LinkedHashSet<>(emptySet())); - - expect(m_masterHostResolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER")).andReturn(null).anyTimes(); - expect(m_masterHostResolver.getMasterAndHosts("HDFS", "NAMENODE")).andReturn(type).anyTimes(); - expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes(); - replay(m_masterHostResolver); - - UpgradeContext context = getMockUpgradeContext(c, Direction.DOWNGRADE, UpgradeType.ROLLING, - repositoryVersion2200); - - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_direction")); - UpgradePack upgrade = upgrades.get("upgrade_direction"); - assertNotNull(upgrade); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(2, groups.size()); - - UpgradeGroupHolder group = groups.get(0); - assertEquals(1, group.items.size()); - assertEquals("PRE_POST_CLUSTER", group.name); - - group = groups.get(1); - assertEquals("POST_CLUSTER", group.name); - assertEquals(3, group.items.size()); - - - StageWrapper stage = group.items.get(1); - assertEquals("NameNode Finalize", stage.getText()); - assertEquals(1, stage.getTasks().size()); - TaskWrapper task = stage.getTasks().get(0); - assertEquals(1, task.getHosts().size()); - } - - @Test - public void testResolverWithFailedUpgrade() throws Exception { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - StackId stackId = new StackId("HDP-2.1.1"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // !!! add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "ZOOKEEPER", "ZOOKEEPER", repositoryVersion2110)); - - Service s = c.getService("ZOOKEEPER"); - ServiceComponent sc = s.addServiceComponent("ZOOKEEPER_SERVER"); - - ServiceComponentHost sch1 = sc.addServiceComponentHost("h1"); - sch1.setVersion(repositoryVersion2110.getVersion()); - - ServiceComponentHost sch2 = sc.addServiceComponentHost("h2"); - sch2.setVersion(repositoryVersion2110.getVersion()); - - List schs = c.getServiceComponentHosts("ZOOKEEPER", "ZOOKEEPER_SERVER"); - assertEquals(2, schs.size()); - - UpgradeContext context = getMockUpgradeContextNoReplay(c, Direction.UPGRADE, - UpgradeType.HOST_ORDERED, repositoryVersion2110); - - MasterHostResolver resolver = new MasterHostResolver(c, m_configHelper, context); - expect(context.getResolver()).andReturn(resolver).anyTimes(); - replay(context); - - HostsType ht = resolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER"); - assertEquals(0, ht.getHosts().size()); - - // !!! if one of them is failed, it should be scheduled - sch2.setUpgradeState(UpgradeState.FAILED); - - ht = resolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER"); - - assertEquals(1, ht.getHosts().size()); - assertEquals("h2", ht.getHosts().iterator().next()); - } - - /** - * Test that MasterHostResolver is case-insensitive even if configs have hosts in upper case for NameNode. - * @throws Exception - */ - @Test - public void testResolverCaseInsensitive() throws Exception { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - String version = "2.1.1.0-1234"; - - StackId stackId = new StackId("HDP-2.1.1"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repositoryVersion211 = helper.getOrCreateRepositoryVersion(stackId, - version); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // Add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HDFS", "HDFS", repositoryVersion211)); - - Service s = c.getService("HDFS"); - ServiceComponent sc = s.addServiceComponent("NAMENODE"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - - List schs = c.getServiceComponentHosts("HDFS", "NAMENODE"); - assertEquals(2, schs.size()); - - setConfigMocks(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.internal.nameservices")).andReturn("ha").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes(); - - // Notice that these names are all caps. - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes(); - replay(m_configHelper); - - UpgradeContext context = getMockUpgradeContextNoReplay(c, Direction.UPGRADE, - UpgradeType.NON_ROLLING, repositoryVersion211); - - // use a "real" master host resolver here so that we can actually test MM - MasterHostResolver mhr = new MockMasterHostResolver(c, m_configHelper, context); - - expect(context.getResolver()).andReturn(mhr).anyTimes(); - replay(context); - - - HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE"); - assertNotNull(ht.getMasters()); - assertNotNull(ht.getSecondaries()); - assertEquals(2, ht.getHosts().size()); - - // Should be stored in lowercase. - assertTrue(ht.getHosts().contains("h1")); - assertTrue(ht.getHosts().contains("h1")); - } - - @Test - public void testResolverBadJmx() throws Exception { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - String version = "2.1.1.0-1234"; - - StackId stackId = new StackId("HDP-2.1.1"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repositoryVersion211 = helper.getOrCreateRepositoryVersion(stackId, version); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // Add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HDFS", "HDFS", repositoryVersion211)); - - Service s = c.getService("HDFS"); - ServiceComponent sc = s.addServiceComponent("NAMENODE"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - - List schs = c.getServiceComponentHosts("HDFS", "NAMENODE"); - assertEquals(2, schs.size()); - - setConfigMocks(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.internal.nameservices")).andReturn("ha").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes(); - - // Notice that these names are all caps. - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes(); - expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes(); - replay(m_configHelper); - - UpgradeContext context = getMockUpgradeContextNoReplay(c, Direction.UPGRADE, - UpgradeType.NON_ROLLING, repositoryVersion211); - - // use a "real" master host resolver here so that we can actually test MM - MasterHostResolver mhr = new BadMasterHostResolver(c, m_configHelper, context); - - expect(context.getResolver()).andReturn(mhr).anyTimes(); - replay(context); - - HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE"); - assertNotNull(ht.getMasters()); - assertNotNull(ht.getSecondaries()); - assertEquals(2, ht.getHosts().size()); - - // Should be stored in lowercase. - assertTrue(ht.getHosts().contains("h1")); - assertTrue(ht.getHosts().contains("h2")); - } - - - /** - * Tests that advanced {@link Grouping} instances like {@link StopGrouping} - * work with rolling upgrade packs. - * - * @throws Exception - */ - @Test - public void testRollingUpgradesCanUseAdvancedGroupings() throws Exception { - final String clusterName = "c1"; - final StackId sourceStackId = new StackId("HDP", "2.1.1"); - final StackId targetStackId = new StackId("HDP", "2.2.0"); - final Direction upgradeDirection = Direction.UPGRADE; - final UpgradeType upgradeType = UpgradeType.ROLLING; - - Cluster cluster = makeCluster(); - - // grab the right pack - String preferredUpgradePackName = "upgrade_grouping_rolling"; - UpgradePack upgradePack = m_upgradeHelper.suggestUpgradePack(clusterName, sourceStackId, - targetStackId, upgradeDirection, upgradeType, preferredUpgradePackName); - - assertEquals(upgradeType, upgradePack.getType()); - - // get an upgrade - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING, - repositoryVersion2210, RepositoryType.STANDARD, Collections.singleton("ZOOKEEPER")); - - List groupings = upgradePack.getGroups(Direction.UPGRADE); - assertEquals(2, groupings.size()); - assertEquals("STOP_ZOOKEEPER", groupings.get(0).name); - assertEquals("RESTART_ZOOKEEPER", groupings.get(1).name); - - List groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertEquals(2, groups.size()); - - assertEquals("STOP_ZOOKEEPER", groups.get(0).name); - assertEquals("RESTART_ZOOKEEPER", groups.get(1).name); - - // STOP_ZOOKEEPER GROUP - UpgradeGroupHolder group = groups.get(0); - - // Check that the upgrade framework properly expanded the STOP grouping into - // STOP tasks - assertEquals("Stopping ZooKeeper Server on h1 (Batch 1 of 3)", group.items.get(0).getText()); - } - - @Test - public void testOrchestrationNoServerSideOnDowngrade() throws Exception { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - String version = "2.1.1.0-1234"; - StackId stackId = new StackId("HDP-2.1.1"); - StackId stackId2 = new StackId("HDP-2.2.0"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repoVersion211 = helper.getOrCreateRepositoryVersion(stackId, - version); - - RepositoryVersionEntity repoVersion220 = helper.getOrCreateRepositoryVersion(stackId2, "2.2.0"); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // !!! add storm - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "STORM", "STORM", repoVersion211)); - - Service s = c.getService("STORM"); - ServiceComponent sc = s.addServiceComponent("NIMBUS"); - ServiceComponentHost sch1 = sc.addServiceComponentHost("h1"); - ServiceComponentHost sch2 = sc.addServiceComponentHost("h2"); - - UpgradePack upgradePack = new UpgradePack() { - @Override - public List getGroups(Direction direction) { - - Grouping g = new Grouping(); - - OrderService orderService = new OrderService(); - orderService.serviceName = "STORM"; - orderService.components = singletonList("NIMBUS"); - - g.name = "GROUP1"; - g.title = "Nimbus Group"; - g.services.add(orderService); - - return Lists.newArrayList(g); - } - - @Override - public Map> getTasks() { - ManualTask mt = new ManualTask(); - mt.messages = Lists.newArrayList("My New Message"); - - ProcessingComponent pc = new ProcessingComponent(); - pc.name = "NIMBUS_MESSAGE"; - pc.preTasks = Lists.newArrayList(mt); - - return Collections.singletonMap("STORM", Collections.singletonMap("NIMBUS", pc)); - } - - }; - - UpgradeContext context = getMockUpgradeContextNoReplay(c, Direction.UPGRADE, - UpgradeType.NON_ROLLING, repoVersion220); - - // use a "real" master host resolver here so that we can actually test MM - MasterHostResolver masterHostResolver = new MasterHostResolver(c, m_configHelper, context); - - expect(context.getResolver()).andReturn(masterHostResolver).anyTimes(); - replay(context); - - List groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertEquals(1, groups.size()); - - sch1.setVersion(repoVersion211.getVersion()); - sch2.setVersion(repoVersion211.getVersion()); - - context = getMockUpgradeContextNoReplay(c, Direction.DOWNGRADE, UpgradeType.NON_ROLLING, - repoVersion211); - - // use a "real" master host resolver here so that we can actually test MM - masterHostResolver = new MasterHostResolver(c, m_configHelper, context); - - expect(context.getResolver()).andReturn(masterHostResolver).anyTimes(); - replay(context); - - groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertTrue(groups.isEmpty()); - } - - @Test - public void testMultipleServerTasks() throws Exception { - - // !!! make a two node cluster with just ZK - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - String version = "2.1.1.0-1234"; - StackId stackId = new StackId("HDP-2.1.1"); - StackId stackId2 = new StackId("HDP-2.2.0"); - - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(stackId, - version); - - RepositoryVersionEntity repoVersion220 = helper.getOrCreateRepositoryVersion(stackId2, "2.2.0"); - - helper.getOrCreateRepositoryVersion(stackId2, UPGRADE_VERSION); - - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // !!! add services - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "ZOOKEEPER", "ZOOKEEPER", repositoryVersion)); - - Service s = c.getService("ZOOKEEPER"); - ServiceComponent sc = s.addServiceComponent("ZOOKEEPER_SERVER"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - - sc = s.addServiceComponent("ZOOKEEPER_CLIENT"); - sc.addServiceComponentHost("h1"); - sc.addServiceComponentHost("h2"); - - EasyMock.reset(m_masterHostResolver); - - expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes(); - - HostsType type = HostsType.normal("h1", "h2"); - expect(m_masterHostResolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER")).andReturn(type).anyTimes(); - - type = HostsType.normal("h1", "h2"); - expect(m_masterHostResolver.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_CLIENT")).andReturn(type).anyTimes(); - - - replay(m_masterHostResolver); - - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - - ServiceInfo si = ambariMetaInfo.getService("HDP", "2.1.1", "ZOOKEEPER"); - si.setDisplayName("Zk"); - ComponentInfo ci = si.getComponentByName("ZOOKEEPER_SERVER"); - ci.setDisplayName("ZooKeeper1 Server2"); - - UpgradePack upgrade = upgrades.get("upgrade_multi_server_tasks"); - assertNotNull(upgrade); - - UpgradeContext context = getMockUpgradeContext(c, Direction.UPGRADE, UpgradeType.NON_ROLLING, - repoVersion220); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(2, groups.size()); - - - // zk server as a colocated grouping first. XML says to run a manual, 2 configs, and an execute - UpgradeGroupHolder group1 = groups.get(0); - assertEquals(7, group1.items.size()); - - // Stage 1. manual, 2 configs, execute - assertEquals(4, group1.items.get(0).getTasks().size()); - TaskWrapper taskWrapper = group1.items.get(0).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.MANUAL, taskWrapper.getTasks().get(0).getType()); - - taskWrapper = group1.items.get(0).getTasks().get(1); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.CONFIGURE, taskWrapper.getTasks().get(0).getType()); - - taskWrapper = group1.items.get(0).getTasks().get(2); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.CONFIGURE, taskWrapper.getTasks().get(0).getType()); - - taskWrapper = group1.items.get(0).getTasks().get(3); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.EXECUTE, taskWrapper.getTasks().get(0).getType()); - - // Stage 2. restart for h1 - assertEquals(1, group1.items.get(1).getTasks().size()); - taskWrapper = group1.items.get(1).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.RESTART, taskWrapper.getTasks().get(0).getType()); - assertTrue(taskWrapper.getHosts().contains("h1")); - - // Stage 3. service check - assertEquals(1, group1.items.get(2).getTasks().size()); - taskWrapper = group1.items.get(2).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.SERVICE_CHECK, taskWrapper.getTasks().get(0).getType()); - - // stage 4. manual step for validation - assertEquals(1, group1.items.get(3).getTasks().size()); - taskWrapper = group1.items.get(3).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.MANUAL, taskWrapper.getTasks().get(0).getType()); - - // Stage 5. repeat execute as it's not a server-side task. no configure or manual tasks - assertEquals(1, group1.items.get(4).getTasks().size()); - taskWrapper = group1.items.get(4).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.EXECUTE, taskWrapper.getTasks().get(0).getType()); - - // Stage 6. restart for h2. - assertEquals(1, group1.items.get(5).getTasks().size()); - taskWrapper = group1.items.get(5).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.RESTART, taskWrapper.getTasks().get(0).getType()); - assertTrue(taskWrapper.getHosts().contains("h2")); - - // Stage 7. service check - assertEquals(1, group1.items.get(6).getTasks().size()); - taskWrapper = group1.items.get(6).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.SERVICE_CHECK, taskWrapper.getTasks().get(0).getType()); - - - // zk client - UpgradeGroupHolder group2 = groups.get(1); - assertEquals(5, group2.items.size()); - - // Stage 1. Configure - assertEquals(1, group2.items.get(0).getTasks().size()); - taskWrapper = group2.items.get(0).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.CONFIGURE, taskWrapper.getTasks().get(0).getType()); - - // Stage 2. Custom class - assertEquals(1, group2.items.get(1).getTasks().size()); - taskWrapper = group2.items.get(1).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.SERVER_ACTION, taskWrapper.getTasks().get(0).getType()); - - // Stage 3. Restart client on h1 - assertEquals(1, group2.items.get(2).getTasks().size()); - taskWrapper = group2.items.get(2).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.RESTART, taskWrapper.getTasks().get(0).getType()); - - // Stage 4. Restart client on h2 (no configure or custom class) - assertEquals(1, group2.items.get(3).getTasks().size()); - taskWrapper = group2.items.get(3).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.RESTART, taskWrapper.getTasks().get(0).getType()); - - // Stage 5. service check - assertEquals(1, group2.items.get(4).getTasks().size()); - taskWrapper = group2.items.get(4).getTasks().get(0); - assertEquals(1, taskWrapper.getTasks().size()); - assertEquals(Task.Type.SERVICE_CHECK, taskWrapper.getTasks().get(0).getType()); - - } - - - - - /** - * Tests {@link UpgradeType#HOST_ORDERED}, specifically that the orchestration - * can properly expand the single {@link HostOrderGrouping} and create the - * correct stages based on the dependencies of the components. - * - * @throws Exception - */ - @Test - public void testHostGroupingOrchestration() throws Exception { - Clusters clusters = injector.getInstance(Clusters.class); - ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class); - ServiceGroupFactory serviceGroupFactory = injector.getInstance(ServiceGroupFactory.class); - - String clusterName = "c1"; - - String version = "2.1.1.0-1234"; - StackId stackId = new StackId("HDP-2.1.1"); - StackId stackId2 = new StackId("HDP-2.2.0"); - clusters.addCluster(clusterName, stackId); - Cluster c = clusters.getCluster(clusterName); - - RepositoryVersionEntity repoVersion211 = helper.getOrCreateRepositoryVersion(stackId, version); - - RepositoryVersionEntity repoVersion220 = helper.getOrCreateRepositoryVersion(stackId2, "2.2.0"); - - // create 2 hosts - for (int i = 0; i < 2; i++) { - String hostName = "h" + (i+1); - clusters.addHost(hostName); - Host host = clusters.getHost(hostName); - - Map hostAttributes = new HashMap<>(); - hostAttributes.put("os_family", "redhat"); - hostAttributes.put("os_release_version", "6"); - - host.setHostAttributes(hostAttributes); - - clusters.mapHostToCluster(hostName, clusterName); - } - - // add ZK Server to both hosts, and then Nimbus to only 1 - this will test - // how the HOU breaks out dependencies into stages - ServiceGroup serviceGroup = serviceGroupFactory.createNew(c, "service_group", new HashSet()); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "ZOOKEEPER", "ZOOKEEPER", repoVersion211)); - c.addService(serviceFactory.createNew(c, serviceGroup, new ArrayList(), "HBASE", "HBASE", repoVersion211)); - Service zookeeper = c.getService("ZOOKEEPER"); - Service hbase = c.getService("HBASE"); - ServiceComponent zookeeperServer = zookeeper.addServiceComponent("ZOOKEEPER_SERVER"); - ServiceComponentHost zookeeperServer1 = zookeeperServer.addServiceComponentHost("h1"); - ServiceComponentHost zookeeperServer2 = zookeeperServer.addServiceComponentHost("h2"); - ServiceComponent hbaseMaster = hbase.addServiceComponent("HBASE_MASTER"); - ServiceComponentHost hbaseMaster1 = hbaseMaster.addServiceComponentHost("h1"); - - // !!! make a custom grouping - HostOrderItem hostItem = new HostOrderItem(HostOrderActionType.HOST_UPGRADE, - Lists.newArrayList("h1", "h2")); - - HostOrderItem checkItem = new HostOrderItem(HostOrderActionType.SERVICE_CHECK, - Lists.newArrayList("ZOOKEEPER", "HBASE")); - - Grouping g = new HostOrderGrouping(); - ((HostOrderGrouping) g).setHostOrderItems(Lists.newArrayList(hostItem, checkItem)); - g.title = "Some Title"; - - UpgradePack upgradePack = new UpgradePack(); - - // !!! set the groups directly; allow the logic in getGroups(Direction) to happen - Field field = UpgradePack.class.getDeclaredField("groups"); - field.setAccessible(true); - field.set(upgradePack, Lists.newArrayList(g)); - - field = UpgradePack.class.getDeclaredField("type" ); - field.setAccessible(true); - field.set(upgradePack, UpgradeType.HOST_ORDERED); - - UpgradeContext context = getMockUpgradeContextNoReplay(c, Direction.UPGRADE, - UpgradeType.HOST_ORDERED, repoVersion220); - - MasterHostResolver resolver = new MasterHostResolver(c, m_configHelper, context); - expect(context.getResolver()).andReturn(resolver).anyTimes(); - replay(context); - - - List groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertEquals(1, groups.size()); - - UpgradeGroupHolder holder = groups.get(0); - assertEquals(9, holder.items.size()); - - for (int i = 0; i < 7; i++) { - StageWrapper w = holder.items.get(i); - if (i == 0 || i == 4) { - assertEquals(StageWrapper.Type.STOP, w.getType()); - } else if (i == 1 || i == 5) { - assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, w.getType()); - assertEquals(1, w.getTasks().size()); - assertEquals(1, w.getTasks().get(0).getTasks().size()); - Task t = w.getTasks().get(0).getTasks().get(0); - assertEquals(ManualTask.class, t.getClass()); - ManualTask mt = (ManualTask) t; - assertNotNull(mt.structuredOut); - assertTrue(mt.structuredOut.contains("type")); - assertTrue(mt.structuredOut.contains(HostOrderItem.HostOrderActionType.HOST_UPGRADE.toString())); - assertTrue(mt.structuredOut.contains("host")); - assertTrue(mt.structuredOut.contains(i == 1 ? "h1" : "h2")); - } else { - assertEquals(StageWrapper.Type.RESTART, w.getType()); - } - } - - assertEquals(StageWrapper.Type.SERVICE_CHECK, holder.items.get(7).getType()); - assertEquals(StageWrapper.Type.SERVICE_CHECK, holder.items.get(8).getType()); - - // !!! test downgrade when all host components have failed - zookeeperServer1.setVersion(repoVersion211.getVersion()); - zookeeperServer2.setVersion(repoVersion211.getVersion()); - hbaseMaster1.setVersion(repoVersion211.getVersion()); - - context = getMockUpgradeContextNoReplay(c, Direction.DOWNGRADE, UpgradeType.HOST_ORDERED, - repoVersion211); - - resolver = new MasterHostResolver(c, m_configHelper, context); - expect(context.getResolver()).andReturn(resolver).anyTimes(); - replay(context); - - groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertEquals(1, groups.size()); - assertEquals(2, groups.get(0).items.size()); - - // !!! test downgrade when one of the hosts had failed - zookeeperServer1.setVersion(repoVersion211.getVersion()); - zookeeperServer2.setVersion(repoVersion220.getVersion()); - hbaseMaster1.setVersion(repoVersion211.getVersion()); - - context = getMockUpgradeContextNoReplay(c, Direction.DOWNGRADE, UpgradeType.HOST_ORDERED, - repoVersion211); - - resolver = new MasterHostResolver(c, m_configHelper, context); - expect(context.getResolver()).andReturn(resolver).anyTimes(); - replay(context); - - groups = m_upgradeHelper.createSequence(upgradePack, context); - - assertEquals(1, groups.size()); - assertEquals(5, groups.get(0).items.size()); - } - - /** - * Tests that the {@link SecurityCondition} element correctly restricts the groups in - * an upgrade. - * - * @throws Exception - */ - @Test - public void testUpgradeConditions() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - assertTrue(upgrades.containsKey("upgrade_test_conditions")); - UpgradePack upgrade = upgrades.get("upgrade_test_conditions"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING); - - // initially, no conditions should be met, so only 1 group should be - // available - List groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(1, groups.size()); - - // from that 1 group, only 1 task is condition-less - List stageWrappers = groups.get(0).items; - assertEquals(1, stageWrappers.size()); - assertEquals(1, stageWrappers.get(0).getTasks().size()); - - // set the configuration property and try again - Map fooConfigs = new HashMap<>(); - fooConfigs.put("foo-property", "foo-value"); - ConfigurationRequest configurationRequest = new ConfigurationRequest(); - configurationRequest.setClusterName(cluster.getClusterName()); - configurationRequest.setType("foo-site"); - configurationRequest.setVersionTag("version1"); - configurationRequest.setProperties(fooConfigs); - - final ClusterRequest clusterRequest = new ClusterRequest(cluster.getClusterId(), - cluster.getClusterName(), cluster.getDesiredStackVersion().getStackVersion(), null); - - clusterRequest.setDesiredConfig(singletonList(configurationRequest)); - m_managementController.updateClusters(Sets.newHashSet(clusterRequest), null); - - // the config condition should now be set - groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(2, groups.size()); - assertEquals("ZOOKEEPER_CONFIG_CONDITION_TEST", groups.get(0).name); - - // now change the cluster security so the other conditions come back too - cluster.setSecurityType(SecurityType.KERBEROS); - - groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(4, groups.size()); - } - - /** - * Tests merging configurations between existing and new stack values on - * upgrade. - * - * @throws Exception - */ - @Test - public void testMergeConfigurations() throws Exception { - RepositoryVersionEntity repoVersion211 = createNiceMock(RepositoryVersionEntity.class); - RepositoryVersionEntity repoVersion220 = createNiceMock(RepositoryVersionEntity.class); - - StackId stack211 = new StackId("HDP-2.1.1"); - StackId stack220 = new StackId("HDP-2.2.0"); - - String version211 = "2.1.1.0-1234"; - String version220 = "2.2.0.0-1234"; - - expect(repoVersion211.getStackId()).andReturn(stack211).atLeastOnce(); - expect(repoVersion211.getVersion()).andReturn(version211).atLeastOnce(); - - expect(repoVersion220.getStackId()).andReturn(stack220).atLeastOnce(); - expect(repoVersion220.getVersion()).andReturn(version220).atLeastOnce(); - - Map> stack211Configs = new HashMap<>(); - Map stack211FooType = new HashMap<>(); - Map stack211BarType = new HashMap<>(); - Map stack211BazType = new HashMap<>(); - stack211Configs.put("foo-site", stack211FooType); - stack211Configs.put("bar-site", stack211BarType); - stack211Configs.put("baz-site", stack211BazType); - stack211FooType.put("1", "one"); - stack211FooType.put("1A", "one-A"); - stack211BarType.put("2", "two"); - stack211BazType.put("3", "three"); - - Map> stack220Configs = new HashMap<>(); - Map stack220FooType = new HashMap<>(); - Map stack220BazType = new HashMap<>(); - stack220Configs.put("foo-site", stack220FooType); - stack220Configs.put("baz-site", stack220BazType); - stack220FooType.put("1", "one-new"); - stack220FooType.put("1A1", "one-A-one"); - stack220BazType.put("3", "three-new"); - - Map existingFooType = new HashMap<>(); - Map existingBarType = new HashMap<>(); - Map existingBazType = new HashMap<>(); - - ClusterConfigEntity fooConfigEntity = createNiceMock(ClusterConfigEntity.class); - ClusterConfigEntity barConfigEntity = createNiceMock(ClusterConfigEntity.class); - ClusterConfigEntity bazConfigEntity = createNiceMock(ClusterConfigEntity.class); - - expect(fooConfigEntity.getType()).andReturn("foo-site"); - expect(barConfigEntity.getType()).andReturn("bar-site"); - expect(bazConfigEntity.getType()).andReturn("baz-site"); - - Config fooConfig = createNiceMock(Config.class); - Config barConfig = createNiceMock(Config.class); - Config bazConfig = createNiceMock(Config.class); - - existingFooType.put("1", "one"); - existingFooType.put("1A", "one-A"); - existingBarType.put("2", "two"); - existingBazType.put("3", "three-changed"); - - expect(fooConfig.getType()).andReturn("foo-site").atLeastOnce(); - expect(barConfig.getType()).andReturn("bar-site").atLeastOnce(); - expect(bazConfig.getType()).andReturn("baz-site").atLeastOnce(); - expect(fooConfig.getProperties()).andReturn(existingFooType); - expect(barConfig.getProperties()).andReturn(existingBarType); - expect(bazConfig.getProperties()).andReturn(existingBazType); - - Map desiredConfigurations = new HashMap<>(); - desiredConfigurations.put("foo-site", null); - desiredConfigurations.put("bar-site", null); - desiredConfigurations.put("baz-site", null); - - Service zookeeper = createNiceMock(Service.class); - expect(zookeeper.getName()).andReturn("ZOOKEEPER").anyTimes(); - expect(zookeeper.getServiceType()).andReturn("ZOOKEEPER").anyTimes(); - expect(zookeeper.getServiceComponents()).andReturn( - new HashMap<>()).once(); - zookeeper.setDesiredRepositoryVersion(repoVersion220); - expectLastCall().once(); - - Cluster cluster = createNiceMock(Cluster.class); - expect(cluster.getCurrentStackVersion()).andReturn(stack211).atLeastOnce(); - expect(cluster.getDesiredStackVersion()).andReturn(stack220); - expect(cluster.getDesiredConfigs()).andReturn(desiredConfigurations); - expect(cluster.getDesiredConfigByType("foo-site")).andReturn(fooConfig); - expect(cluster.getDesiredConfigByType("bar-site")).andReturn(barConfig); - expect(cluster.getDesiredConfigByType("baz-site")).andReturn(bazConfig); - expect(cluster.getService("ZOOKEEPER")).andReturn(zookeeper).anyTimes(); - expect(cluster.getService(anyString(), eq("ZOOKEEPER"))).andReturn(zookeeper).anyTimes(); - expect(cluster.getDesiredConfigByType("foo-type")).andReturn(fooConfig); - expect(cluster.getDesiredConfigByType("bar-type")).andReturn(barConfig); - expect(cluster.getDesiredConfigByType("baz-type")).andReturn(bazConfig); - - // setup the config helper for placeholder resolution - @SuppressWarnings("unchecked") - Provider configHelperProvider = EasyMock.createNiceMock(Provider.class); - ConfigHelper configHelper = EasyMock.createNiceMock(ConfigHelper.class); - - expect(configHelperProvider.get()).andStubReturn(configHelper); - - expect(configHelper.getDefaultProperties(stack211, "ZOOKEEPER")).andReturn( - stack211Configs).anyTimes(); - - expect(configHelper.getDefaultProperties(stack220, "ZOOKEEPER")).andReturn( - stack220Configs).anyTimes(); - - Capture>> expectedConfigurationsCapture = EasyMock.newCapture(); - - configHelper.createConfigTypes(EasyMock.anyObject(Cluster.class), - EasyMock.anyObject(StackId.class), EasyMock.anyObject(AmbariManagementController.class), - EasyMock.capture(expectedConfigurationsCapture), EasyMock.anyObject(String.class), - EasyMock.anyObject(String.class)); - - expectLastCall().once(); - EasyMock.replay(configHelperProvider, configHelper); - - // mock the service config DAO and replay it - ServiceConfigEntity zookeeperServiceConfig = createNiceMock(ServiceConfigEntity.class); - expect(zookeeperServiceConfig.getClusterConfigEntities()).andReturn( - Lists.newArrayList(fooConfigEntity, barConfigEntity, bazConfigEntity)); - - ServiceConfigDAO serviceConfigDAOMock; - serviceConfigDAOMock = EasyMock.createNiceMock(ServiceConfigDAO.class); - - List latestServiceConfigs = Lists.newArrayList(zookeeperServiceConfig); - expect(serviceConfigDAOMock.getLastServiceConfigsForService(EasyMock.anyLong(), - EasyMock.anyLong())).andReturn(latestServiceConfigs).once(); - - replay(serviceConfigDAOMock); - - Map upgradePacks = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - UpgradePack upgradePack = upgradePacks.get("upgrade_to_new_stack"); - - UpgradeContext context = createNiceMock(UpgradeContext.class); - expect(context.getCluster()).andReturn(cluster).atLeastOnce(); - expect(context.getType()).andReturn(UpgradeType.ROLLING).atLeastOnce(); - expect(context.getDirection()).andReturn(Direction.UPGRADE).atLeastOnce(); - expect(context.getRepositoryVersion()).andReturn(repoVersion220).anyTimes(); - expect(context.getSupportedServices()).andReturn(Sets.newHashSet("ZOOKEEPER")).atLeastOnce(); - expect(context.getSourceRepositoryVersion(anyString())).andReturn(repoVersion211).atLeastOnce(); - expect(context.getTargetRepositoryVersion(anyString())).andReturn(repoVersion220).atLeastOnce(); - expect(context.getOrchestrationType()).andReturn(RepositoryType.STANDARD).anyTimes(); - expect(context.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); - expect(context.getHostRoleCommandFactory()).andStubReturn(injector.getInstance(HostRoleCommandFactory.class)); - expect(context.getRoleGraphFactory()).andStubReturn(injector.getInstance(RoleGraphFactory.class)); - expect(context.getUpgradePack()).andReturn(upgradePack).atLeastOnce(); - - replayAll(); - - UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); - upgradeHelper.m_serviceConfigDAO = serviceConfigDAOMock; - upgradeHelper.m_configHelperProvider = configHelperProvider; - upgradeHelper.updateDesiredRepositoriesAndConfigs(context); - - Map> expectedConfigurations = expectedConfigurationsCapture.getValue(); - Map expectedFooType = expectedConfigurations.get("foo-site"); - Map expectedBarType = expectedConfigurations.get("bar-site"); - Map expectedBazType = expectedConfigurations.get("baz-site"); - - // As the upgrade pack did not have any Flume updates, its configs should - // not be updated. - assertEquals(3, expectedConfigurations.size()); - - // the really important values are one-new and three-changed; one-new - // indicates that the new stack value is changed since it was not customized - // while three-changed represents that the customized value was preserved - // even though the stack value changed - assertEquals("one-new", expectedFooType.get("1")); - assertEquals("one-A", expectedFooType.get("1A")); - assertEquals("two", expectedBarType.get("2")); - assertEquals("three-changed", expectedBazType.get("3")); - } - - @Test - public void testMergeConfigurationsWithClusterEnv() throws Exception { - Cluster cluster = makeCluster(true); - - StackId oldStack = cluster.getDesiredStackVersion(); - StackId newStack = new StackId("HDP-2.5.0"); - - ConfigFactory cf = injector.getInstance(ConfigFactory.class); - - Config clusterEnv = cf.createNew(cluster, "cluster-env", "version1", - ImmutableMap.builder().put("a", "b").build(), - Collections.emptyMap()); - - Config zooCfg = cf.createNew(cluster, "zoo.cfg", "version1", - ImmutableMap.builder().put("c", "d").build(), - Collections.emptyMap()); - - cluster.addDesiredConfig("admin", Sets.newHashSet(clusterEnv, zooCfg)); - - Map> stackMap = new HashMap<>(); - stackMap.put("cluster-env", new HashMap<>()); - stackMap.put("hive-site", new HashMap<>()); - - final Map clusterEnvMap = new HashMap<>(); - - Capture captureCluster = Capture.newInstance(); - Capture captureStackId = Capture.newInstance(); - Capture captureAmc = Capture.newInstance(); - - Capture>> cap = new Capture>>() { - @Override - public void setValue(Map> value) { - if (value.containsKey("cluster-env")) { - clusterEnvMap.putAll(value.get("cluster-env")); - } - } - }; - - Capture captureUsername = Capture.newInstance(); - Capture captureNote = Capture.newInstance(); - - EasyMock.reset(m_configHelper); - expect(m_configHelper.getDefaultProperties(oldStack, "HIVE")).andReturn(stackMap).atLeastOnce(); - expect(m_configHelper.getDefaultProperties(newStack, "HIVE")).andReturn(stackMap).atLeastOnce(); - expect(m_configHelper.getDefaultProperties(oldStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); - expect(m_configHelper.getDefaultProperties(newStack, "ZOOKEEPER")).andReturn(stackMap).atLeastOnce(); - m_configHelper.createConfigTypes( - EasyMock.capture(captureCluster), - EasyMock.capture(captureStackId), - EasyMock.capture(captureAmc), - EasyMock.capture(cap), - - EasyMock.capture(captureUsername), - EasyMock.capture(captureNote)); - expectLastCall().atLeastOnce(); - - replay(m_configHelper); - - RepositoryVersionEntity repoVersionEntity = helper.getOrCreateRepositoryVersion(new StackId("HDP-2.5.0"), "2.5.0-1234"); - - Map upgradeRequestMap = new HashMap<>(); - upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); - upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, repoVersionEntity.getId().toString()); - upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_test_HDP-250"); - upgradeRequestMap.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, Boolean.TRUE.toString()); - - UpgradeContextFactory contextFactory = injector.getInstance(UpgradeContextFactory.class); - UpgradeContext context = contextFactory.create(cluster, upgradeRequestMap); - - UpgradeHelper upgradeHelper = injector.getInstance(UpgradeHelper.class); - upgradeHelper.updateDesiredRepositoriesAndConfigs(context); - - assertNotNull(clusterEnvMap); - assertTrue(clusterEnvMap.containsKey("a")); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - @Test - public void testSequentialServiceChecks() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - cluster.deleteService("HDFS"); - cluster.deleteService("YARN"); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, - UpgradeType.ROLLING, repositoryVersion2110); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(5, groups.size()); - - UpgradeGroupHolder serviceCheckGroup = groups.get(2); - assertEquals(ServiceCheckGrouping.class, serviceCheckGroup.groupClass); - assertEquals(3, serviceCheckGroup.items.size()); - - StageWrapper wrapper = serviceCheckGroup.items.get(0); - assertEquals(ServiceCheckGrouping.ServiceCheckStageWrapper.class, wrapper.getClass()); - assertTrue(wrapper.getText().contains("ZooKeeper")); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - @Test - public void testSequentialServiceChecksWithServiceCheckFailure() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - assertNotNull(upgrade); - - // !!! fake skippable so we don't affect other tests - for (Grouping g : upgrade.getAllGroups()) { - if (g.name.equals("SERVICE_CHECK_1") || g.name.equals("SERVICE_CHECK_2")) { - g.skippable = true; - } - } - - Cluster cluster = makeCluster(); - cluster.deleteService("HDFS"); - cluster.deleteService("YARN"); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, UpgradeType.ROLLING, repositoryVersion2110, - RepositoryType.STANDARD, cluster.getServices().keySet(), m_masterHostResolver, false); - expect(context.isServiceCheckFailureAutoSkipped()).andReturn(Boolean.TRUE).atLeastOnce(); - - replay(context); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - assertEquals(5, groups.size()); - - UpgradeGroupHolder serviceCheckGroup = groups.get(2); - assertEquals(ServiceCheckGrouping.class, serviceCheckGroup.groupClass); - assertEquals(4, serviceCheckGroup.items.size()); - - StageWrapper wrapper = serviceCheckGroup.items.get(0); - assertEquals(ServiceCheckGrouping.ServiceCheckStageWrapper.class, wrapper.getClass()); - assertTrue(wrapper.getText().contains("ZooKeeper")); - - wrapper = serviceCheckGroup.items.get(serviceCheckGroup.items.size()-1); - assertTrue(wrapper.getText().equals("Verifying Skipped Failures")); - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - - @Test - public void testPrematureServiceChecks() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - assertNotNull(upgrade); - - Cluster cluster = makeCluster(); - cluster.deleteService("HDFS"); - cluster.deleteService("YARN"); - cluster.deleteService("ZOOKEEPER"); - - UpgradeContext context = getMockUpgradeContext(cluster, Direction.UPGRADE, - UpgradeType.ROLLING, repositoryVersion2110); - - List groups = m_upgradeHelper.createSequence(upgrade, context); - - assertEquals(3, groups.size()); - - for (UpgradeGroupHolder holder : groups) { - assertFalse(ServiceCheckGrouping.class.equals(holder.groupClass)); - } - - // Do stacks cleanup - stackManagerMock.invalidateCurrentPaths(); - ambariMetaInfo.init(); - } - - - /** - * @param cluster - * @param direction - * @param type - * @return - */ - private UpgradeContext getMockUpgradeContext(Cluster cluster, Direction direction, UpgradeType type){ - return getMockUpgradeContext(cluster, direction, type, repositoryVersion2210); - } - - /** - * @param cluster - * @param direction - * @param type - * @return - */ - private UpgradeContext getMockUpgradeContext(Cluster cluster, Direction direction, - UpgradeType type, RepositoryVersionEntity repositoryVersion) { - Set allServices = cluster.getServices().keySet(); - return getMockUpgradeContext(cluster, direction, type, repositoryVersion, - RepositoryType.STANDARD, allServices); - } - - /** - * @param cluster - * @param direction - * @param type - * @return - */ - private UpgradeContext getMockUpgradeContext(Cluster cluster, Direction direction, - UpgradeType type, RepositoryVersionEntity repositoryVersion, RepositoryType repositoryType, - Set services) { - return getMockUpgradeContext(cluster, direction, type, repositoryVersion, - repositoryType, services, m_masterHostResolver, true); - } - - /** - * @param cluster - * @param direction - * @param type - * @return - */ - private UpgradeContext getMockUpgradeContextNoReplay(Cluster cluster, Direction direction, - UpgradeType type, RepositoryVersionEntity repositoryVersion) { - Set allServices = cluster.getServices().keySet(); - - return getMockUpgradeContext(cluster, direction, type, repositoryVersion, - RepositoryType.STANDARD, allServices, null, false); - } - - /** - * @param cluster - * @param direction - * @param type - * @param repositoryType - * @param services - * @return - */ - private UpgradeContext getMockUpgradeContext(Cluster cluster, Direction direction, - UpgradeType type, RepositoryVersionEntity repositoryVersion, final RepositoryType repositoryType, - Set services, MasterHostResolver resolver, boolean replay) { - UpgradeContext context = EasyMock.createNiceMock(UpgradeContext.class); - expect(context.getCluster()).andReturn(cluster).anyTimes(); - expect(context.getType()).andReturn(type).anyTimes(); - expect(context.getDirection()).andReturn(direction).anyTimes(); - expect(context.getRepositoryVersion()).andReturn(repositoryVersion).anyTimes(); - expect(context.getSupportedServices()).andReturn(services).anyTimes(); - expect(context.getOrchestrationType()).andReturn(repositoryType).anyTimes(); - expect(context.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); - expect(context.getHostRoleCommandFactory()).andStubReturn( - injector.getInstance(HostRoleCommandFactory.class)); - expect(context.getRoleGraphFactory()).andStubReturn( - injector.getInstance(RoleGraphFactory.class)); - - // only set this if supplied - if (null != resolver) { - expect(context.getResolver()).andReturn(resolver).anyTimes(); - } - - final Map targetRepositoryVersions = new HashMap<>(); - for( String serviceName : services ){ - targetRepositoryVersions.put(serviceName, repositoryVersion); - } - - final Capture repoVersionServiceName = EasyMock.newCapture(); - expect( - context.getTargetRepositoryVersion(EasyMock.capture(repoVersionServiceName))).andAnswer( - new IAnswer() { - @Override - public RepositoryVersionEntity answer() { - return targetRepositoryVersions.get(repoVersionServiceName.getValue()); - } - }).anyTimes(); - - final Capture serviceNameSupported = EasyMock.newCapture(); - expect(context.isServiceSupported(EasyMock.capture(serviceNameSupported))).andAnswer( - new IAnswer() { - @Override - public Boolean answer() { - return targetRepositoryVersions.containsKey(serviceNameSupported.getValue()); - } - }).anyTimes(); - - - final Map serviceNames = new HashMap<>(); - final Capture serviceDisplayNameArg1 = EasyMock.newCapture(); - final Capture serviceDisplayNameArg2 = EasyMock.newCapture(); - - context.setServiceDisplay(EasyMock.capture(serviceDisplayNameArg1), EasyMock.capture(serviceDisplayNameArg2)); - expectLastCall().andAnswer( - new IAnswer() { - @Override - public Object answer() { - serviceNames.put(serviceDisplayNameArg1.getValue(), serviceDisplayNameArg2.getValue()); - return null; - } - }).anyTimes(); - - - final Map componentNames = new HashMap<>(); - final Capture componentDisplayNameArg1 = EasyMock.newCapture(); - final Capture componentDisplayNameArg2 = EasyMock.newCapture(); - final Capture componentDisplayNameArg3 = EasyMock.newCapture(); - - context.setComponentDisplay(EasyMock.capture(componentDisplayNameArg1), - EasyMock.capture(componentDisplayNameArg2), EasyMock.capture(componentDisplayNameArg3)); - - expectLastCall().andAnswer(new IAnswer() { - @Override - public Object answer() { - componentNames.put( - componentDisplayNameArg1.getValue() + ":" + componentDisplayNameArg2.getValue(), - componentDisplayNameArg3.getValue()); - return null; - } - }).anyTimes(); - - final Capture getServiceDisplayArgument1 = EasyMock.newCapture(); - expect( - context.getServiceDisplay(EasyMock.capture(getServiceDisplayArgument1))).andAnswer( - new IAnswer() { - @Override - public String answer() { - return serviceNames.get(getServiceDisplayArgument1.getValue()); - } - }).anyTimes(); - - final Capture getComponentDisplayArgument1 = EasyMock.newCapture(); - final Capture getComponentDisplayArgument2 = EasyMock.newCapture(); - expect(context.getComponentDisplay(EasyMock.capture(getComponentDisplayArgument1), - EasyMock.capture(getComponentDisplayArgument2))).andAnswer(new IAnswer() { - @Override - public String answer() { - return componentNames.get(getComponentDisplayArgument1.getValue() + ":" - + getComponentDisplayArgument2.getValue()); - } - }).anyTimes(); - - final Capture isScopedCapture = EasyMock.newCapture(); - expect(context.isScoped(EasyMock.capture(isScopedCapture))).andStubAnswer( - new IAnswer() { - @Override - public Boolean answer() throws Throwable { - UpgradeScope scope = isScopedCapture.getValue(); - if (scope == UpgradeScope.ANY) { - return true; - } - - if (scope == UpgradeScope.PARTIAL) { - return repositoryType != RepositoryType.STANDARD; - } - - return repositoryType == RepositoryType.STANDARD; - } - }); - - if (replay) { - replay(context); - } - - return context; - } - - /** - * Extend {@link org.apache.ambari.server.stack.MasterHostResolver} in order - * to overwrite the JMX methods. - */ - private class MockMasterHostResolver extends MasterHostResolver { - - public MockMasterHostResolver(Cluster cluster, ConfigHelper configHelper, UpgradeContext context) { - super(cluster, configHelper, context); - } - - /** - * Mock the call to get JMX Values. - * @param hostname host name - * @param port port number - * @param beanName if asQuery is false, then search for this bean name - * @param attributeName if asQuery is false, then search for this attribute name - * @param asQuery whether to search bean or query - * @param encrypted true if using https instead of http. - * @return - */ - @Override - public String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName, - boolean asQuery, boolean encrypted) { - - if (beanName.equalsIgnoreCase("Hadoop:service=NameNode,name=NameNodeStatus") && attributeName.equalsIgnoreCase("State") && asQuery) { - switch (hostname) { - case "H1": - return Status.ACTIVE.toString(); - case "H2": - return Status.STANDBY.toString(); - case "H3": - return Status.ACTIVE.toString(); - case "H4": - return Status.STANDBY.toString(); - - default: - return "UNKNOWN_NAMENODE_STATUS_FOR_THIS_HOST"; - } - } - return "NOT_MOCKED"; - } - } - - private class MockModule implements Module { - - @Override - public void configure(Binder binder) { - binder.install(new FactoryModuleBuilder().build(UpgradeContextFactory.class)); - binder.bind(ConfigHelper.class).toInstance(m_configHelper); - } - } - - private static class BadMasterHostResolver extends MasterHostResolver { - - public BadMasterHostResolver(Cluster cluster, ConfigHelper configHelper, UpgradeContext context) { - super(cluster, configHelper, context); - } - - @Override - protected String queryJmxBeanValue(String hostname, int port, String beanName, - String attributeName, boolean asQuery, boolean encrypted) { - return null; - } - - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/AlertEventPublisherTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/AlertEventPublisherTest.java index 933693ba9d6..74201e0f9d7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/AlertEventPublisherTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/AlertEventPublisherTest.java @@ -298,7 +298,7 @@ private void installHdfsService() throws Exception { cluster.getCurrentStackVersion(), REPO_VERSION); String serviceName = "HDFS"; - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-1.0"); serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), serviceName, serviceName, repositoryVersion); Service service = cluster.getService(serviceName); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/InitialAlertEventTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/InitialAlertEventTest.java index b1625ff924b..294f6d8dcac 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/InitialAlertEventTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/alerts/InitialAlertEventTest.java @@ -181,7 +181,7 @@ public void testInitialAlertEvent() throws Exception { private void installHdfsService() throws Exception { String serviceName = "HDFS"; - ServiceGroup serviceGroup = m_cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = m_cluster.addServiceGroup("CORE", "HDP-1.0"); m_serviceFactory.createNew(m_cluster, serviceGroup, Collections.emptyList(), serviceName, serviceName, m_repositoryVersion); Service service = m_cluster.getService(serviceName); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterDeadlockTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterDeadlockTest.java index c7c6b7a06fc..9970b9c90ca 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterDeadlockTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterDeadlockTest.java @@ -143,7 +143,7 @@ public void setup() throws Exception { clusters.mapHostToCluster(hostName, "c1"); } - serviceGroup = cluster.addServiceGroup("CORE"); + serviceGroup = cluster.addServiceGroup("CORE", "HDP-1.0"); Service service = installService("HDFS", serviceGroup); addServiceComponent(service, "NAMENODE"); addServiceComponent(service, "DATANODE"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java index 2f52065d1d3..7fbaf6f5404 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java @@ -234,7 +234,7 @@ public void testDeleteService() throws Exception { clusters.mapAndPublishHostsToCluster(Sets.newHashSet(hostName1, hostName2), clusterName); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); ServiceComponent nameNode = hdfs.addServiceComponent("NAMENODE"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index 8ca7dd0cb09..9a7deb6886d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -229,7 +229,7 @@ private void createDefaultCluster(Set hostNames, StackId stackId) throws clusters.mapAndPublishHostsToCluster(hostNames, clusterName); c1 = clusters.getCluster(clusterName); - serviceGroup = c1.addServiceGroup("CORE"); + serviceGroup = c1.addServiceGroup("CORE", stackId.getStackId()); } public ClusterEntity createDummyData() { @@ -334,7 +334,7 @@ private Cluster createClusterForRU(String clusterName, RepositoryVersionEntity r } // Add Services - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); Service s1 = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), "HDFS", "HDFS", repositoryVersion); Service s2 = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), "ZOOKEEPER", "ZOOKEEPER", repositoryVersion); Service s3 = serviceFactory.createNew(cluster, serviceGroup, Collections.emptyList(), "GANGLIA", "GANGLIA", repositoryVersion); @@ -1872,7 +1872,7 @@ public void testTransitionNonReportableHost() throws Exception { RepositoryVersionEntity repositoryVersion = helper.getOrCreateRepositoryVersion(c1); - ServiceGroup serviceGroup = c1.addServiceGroup("CORE"); + ServiceGroup serviceGroup = c1.addServiceGroup("CORE", stackId.getStackId()); Service service = c1.addService(serviceGroup, "ZOOKEEPER", "ZOOKEEPER", repositoryVersion); ServiceComponent sc = service.addServiceComponent("ZOOKEEPER_SERVER"); sc.addServiceComponentHost("h-1"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersDeadlockTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersDeadlockTest.java index f5d7ad0aa00..112166882ab 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersDeadlockTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersDeadlockTest.java @@ -118,7 +118,7 @@ public void setup() throws Exception { helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); // install HDFS - serviceGroup = cluster.addServiceGroup("CORE"); + serviceGroup = cluster.addServiceGroup("CORE", stackId.getStackId()); installService("HDFS", serviceGroup); writerStoppedSignal = new CountDownLatch(NUMBER_OF_THREADS); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java index 8355d36a5f0..849326b6dc2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClustersTest.java @@ -427,7 +427,7 @@ public void testDeleteCluster() throws Exception { // host config override host1.addDesiredConfig(cluster.getClusterId(), true, "_test", config2); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-1.0"); Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS", repositoryVersion); //Assert.assertNotNull(injector.getInstance(ClusterServiceDAO.class).findByClusterAndServiceNames(c1, "HDFS")); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ConcurrentServiceConfigVersionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ConcurrentServiceConfigVersionTest.java index a91319ee2ae..e3d76b323fe 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ConcurrentServiceConfigVersionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ConcurrentServiceConfigVersionTest.java @@ -122,7 +122,7 @@ public void setup() throws Exception { helper.createStack(stackId); clusters.addCluster("c1", stackId); cluster = clusters.getCluster("c1"); - serviceGroup = serviceGroupFactory.createNew(cluster, "test_service_group", new HashSet()); + serviceGroup = serviceGroupFactory.createNew(cluster, "test_service_group", stackId, new HashSet()); String hostName = "c6401.ambari.apache.org"; clusters.addHost(hostName); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java index a62dd6e9672..8289a0db5b6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ServiceComponentHostConcurrentWriteDeadlockTest.java @@ -126,7 +126,7 @@ public void setup() throws Exception { helper.createStack(stackId); clusters.addCluster("c1", stackId); cluster = clusters.getCluster("c1"); - serviceGroup = serviceGroupFactory.createNew(cluster, "test_service_group", new HashSet()); + serviceGroup = serviceGroupFactory.createNew(cluster, "test_service_group", stackId, new HashSet()); Config config1 = configFactory.createNew(cluster, "test-type1", null, new HashMap<>(), new HashMap<>()); Config config2 = configFactory.createNew(cluster, "test-type2", null, new HashMap<>(), new HashMap<>()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradePackTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradePackTest.java deleted file mode 100644 index bdd226fc640..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradePackTest.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.state.stack; - -import static org.apache.ambari.server.state.stack.ConfigUpgradePack.AffectedComponent; -import static org.apache.ambari.server.state.stack.ConfigUpgradePack.AffectedService; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Map; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition; -import org.apache.ambari.server.state.stack.upgrade.TransferOperation; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.google.inject.Guice; -import com.google.inject.Injector; - -/** - * Tests for the config upgrade pack - */ -@Category({ category.StackUpgradeTest.class}) -public class ConfigUpgradePackTest { - - private Injector injector; - private AmbariMetaInfo ambariMetaInfo; - - @Before - public void before() throws Exception { - injector = Guice.createInjector(new InMemoryDefaultTestModule()); - injector.getInstance(GuiceJpaInitializer.class); - - ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); - } - - @After - public void teardown() throws AmbariException, SQLException { - H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); - } - - @Test - public void testMerge() { - // Generate test data - 3 config upgrade packs, 2 services, 2 components, 2 config changes each - ArrayList cups = new ArrayList<>(); - for (int cupIndex = 0; cupIndex < 3; cupIndex++) { - - ArrayList services = new ArrayList<>(); - for (int serviceIndex = 0; serviceIndex < 2; serviceIndex++) { - String serviceName; - if (serviceIndex == 0) { - serviceName = "HDFS"; // For checking merge of existing services - } else { - serviceName = String.format("SOME_SERVICE_%s", cupIndex); - } - ArrayList components = new ArrayList<>(); - for (int componentIndex = 0; componentIndex < 2; componentIndex++) { - String componentName; - if (componentIndex == 0) { - componentName = "NAMENODE"; // For checking merge of existing components - } else { - componentName = "SOME_COMPONENT_" + cupIndex; - } - - ArrayList changeDefinitions = new ArrayList<>(); - for (int changeIndex = 0; changeIndex < 2; changeIndex++) { - String change_id = String.format( - "CHANGE_%s_%s_%s_%s", cupIndex, serviceIndex, componentIndex, changeIndex); - ConfigUpgradeChangeDefinition changeDefinition = new ConfigUpgradeChangeDefinition(); - changeDefinition.id = change_id; - changeDefinitions.add(changeDefinition); - } - AffectedComponent component = new AffectedComponent(); - component.name = componentName; - component.changes = changeDefinitions; - components.add(component); - } - AffectedService service = new AffectedService(); - service.name = serviceName; - service.components = components; - services.add(service); - } - ConfigUpgradePack cupI = new ConfigUpgradePack(); - cupI.services = services; - cups.add(cupI); - } - - // Merge - - ConfigUpgradePack result = ConfigUpgradePack.merge(cups); - - - // Check test results - - assertEquals(result.enumerateConfigChangesByID().entrySet().size(), 24); - - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(0).id, "CHANGE_0_0_0_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(1).id, "CHANGE_0_0_0_1"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(2).id, "CHANGE_1_0_0_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(3).id, "CHANGE_1_0_0_1"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(4).id, "CHANGE_2_0_0_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("NAMENODE").changes.get(5).id, "CHANGE_2_0_0_1"); - - - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_0").changes.get(0).id, "CHANGE_0_0_1_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_0").changes.get(1).id, "CHANGE_0_0_1_1"); - - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_1").changes.get(0).id, "CHANGE_1_0_1_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_1").changes.get(1).id, "CHANGE_1_0_1_1"); - - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_2").changes.get(0).id, "CHANGE_2_0_1_0"); - assertEquals(result.getServiceMap().get("HDFS").getComponentMap().get("SOME_COMPONENT_2").changes.get(1).id, "CHANGE_2_0_1_1"); - - - assertEquals(result.getServiceMap().get("SOME_SERVICE_0").getComponentMap().get("NAMENODE").changes.get(0).id, "CHANGE_0_1_0_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_0").getComponentMap().get("NAMENODE").changes.get(1).id, "CHANGE_0_1_0_1"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_0").getComponentMap().get("SOME_COMPONENT_0").changes.get(0).id, "CHANGE_0_1_1_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_0").getComponentMap().get("SOME_COMPONENT_0").changes.get(1).id, "CHANGE_0_1_1_1"); - - assertEquals(result.getServiceMap().get("SOME_SERVICE_1").getComponentMap().get("NAMENODE").changes.get(0).id, "CHANGE_1_1_0_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_1").getComponentMap().get("NAMENODE").changes.get(1).id, "CHANGE_1_1_0_1"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_1").getComponentMap().get("SOME_COMPONENT_1").changes.get(0).id, "CHANGE_1_1_1_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_1").getComponentMap().get("SOME_COMPONENT_1").changes.get(1).id, "CHANGE_1_1_1_1"); - - assertEquals(result.getServiceMap().get("SOME_SERVICE_2").getComponentMap().get("NAMENODE").changes.get(0).id, "CHANGE_2_1_0_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_2").getComponentMap().get("NAMENODE").changes.get(1).id, "CHANGE_2_1_0_1"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_2").getComponentMap().get("SOME_COMPONENT_2").changes.get(0).id, "CHANGE_2_1_1_0"); - assertEquals(result.getServiceMap().get("SOME_SERVICE_2").getComponentMap().get("SOME_COMPONENT_2").changes.get(1).id, "CHANGE_2_1_1_1"); - - } - - @Test - public void testConfigUpgradeDefinitionParsing() throws Exception { - ConfigUpgradePack cup = ambariMetaInfo.getConfigUpgradePack("HDP", "2.1.1"); - Map changesByID = cup.enumerateConfigChangesByID(); - - ConfigUpgradeChangeDefinition hdp_2_1_1_nm_pre_upgrade = changesByID.get("hdp_2_1_1_nm_pre_upgrade"); - assertEquals("core-site", hdp_2_1_1_nm_pre_upgrade.getConfigType()); - assertEquals(4, hdp_2_1_1_nm_pre_upgrade.getTransfers().size()); - - /* - - - - - important-key - - */ - ConfigUpgradeChangeDefinition.Transfer t1 = hdp_2_1_1_nm_pre_upgrade.getTransfers().get(0); - assertEquals(TransferOperation.COPY, t1.operation); - assertEquals("copy-key", t1.fromKey); - assertEquals("copy-key-to", t1.toKey); - - ConfigUpgradeChangeDefinition.Transfer t2 = hdp_2_1_1_nm_pre_upgrade.getTransfers().get(1); - assertEquals(TransferOperation.COPY, t2.operation); - assertEquals("my-site", t2.fromType); - assertEquals("my-copy-key", t2.fromKey); - assertEquals("my-copy-key-to", t2.toKey); - assertTrue(t2.keepKeys.isEmpty()); - - ConfigUpgradeChangeDefinition.Transfer t3 = hdp_2_1_1_nm_pre_upgrade.getTransfers().get(2); - assertEquals(TransferOperation.MOVE, t3.operation); - assertEquals("move-key", t3.fromKey); - assertEquals("move-key-to", t3.toKey); - - ConfigUpgradeChangeDefinition.Transfer t4 = hdp_2_1_1_nm_pre_upgrade.getTransfers().get(3); - assertEquals(TransferOperation.DELETE, t4.operation); - assertEquals("delete-key", t4.deleteKey); - assertNull(t4.toKey); - assertTrue(t4.preserveEdits); - assertEquals(1, t4.keepKeys.size()); - assertEquals("important-key", t4.keepKeys.get(0)); - - } - -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java deleted file mode 100644 index 2fb0795e07a..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/ConfigUpgradeValidityTest.java +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.state.stack; - -import java.io.File; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.H2DatabaseCleaner; -import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.configuration.Configuration; -import org.apache.ambari.server.controller.internal.UpgradeResourceProvider.ConfigurationPackBuilder; -import org.apache.ambari.server.orm.GuiceJpaInitializer; -import org.apache.ambari.server.orm.InMemoryDefaultTestModule; -import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; -import org.apache.ambari.server.orm.entities.StackEntity; -import org.apache.ambari.server.orm.entities.UpgradeEntity; -import org.apache.ambari.server.orm.entities.UpgradeHistoryEntity; -import org.apache.ambari.server.stack.ModuleFileUnmarshaller; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.StackInfo; -import org.apache.ambari.server.state.UpgradeContext; -import org.apache.ambari.server.state.UpgradeContextFactory; -import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; -import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping; -import org.apache.ambari.server.state.stack.upgrade.ClusterGrouping.ExecuteStage; -import org.apache.ambari.server.state.stack.upgrade.ConfigureTask; -import org.apache.ambari.server.state.stack.upgrade.Direction; -import org.apache.ambari.server.state.stack.upgrade.Grouping; -import org.apache.ambari.server.state.stack.upgrade.Task; -import org.apache.ambari.server.state.stack.upgrade.Task.Type; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.apache.commons.lang.StringUtils; -import org.easymock.EasyMock; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.inject.Guice; -import com.google.inject.Injector; - -import junit.framework.Assert; - -/** - * Tests that for every upgrade pack found, that all referenced configuration - * IDs exist in the {@code config-upgrade.xml} which will be used/created. Also - * ensures that every XML file is valid against its XSD. - */ -@Category({ category.StackUpgradeTest.class}) -public class ConfigUpgradeValidityTest { - - private static final Logger LOG = LoggerFactory.getLogger(ConfigUpgradeValidityTest.class); - - private Injector injector; - private AmbariMetaInfo ambariMetaInfo; - private UpgradeContextFactory upgradeContextFactory; - - private int validatedConfigCount = 0; - - @Before - public void before() throws Exception { - validatedConfigCount = 0; - - // ensure that we use the real stacks - InMemoryDefaultTestModule testModule = new InMemoryDefaultTestModule(); - testModule.getProperties().put(Configuration.METADATA_DIR_PATH.getKey(), - "src/main/resources/stacks"); - - injector = Guice.createInjector(testModule); - injector.getInstance(GuiceJpaInitializer.class); - - ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class); - upgradeContextFactory = injector.getInstance(UpgradeContextFactory.class); - } - - @After - public void teardown() throws AmbariException, SQLException { - H2DatabaseCleaner.clearDatabaseAndStopPersistenceService(injector); - } - - /** - * Tests that every ID specified in an upgrade pack actually exists in the - * created {@link ConfigUpgradePack}. - * - * @throws Exception - */ - @Test - public void testConfigurationDefinitionsExist() throws Exception { - Collection stacks = ambariMetaInfo.getStacks(); - Assert.assertFalse(stacks.isEmpty()); - - Cluster cluster = EasyMock.createNiceMock(Cluster.class); - - for( StackInfo stack : stacks ){ - if (!stack.isActive()) { - LOG.info("Skipping configuration validity test for {}", new StackId(stack)); - continue; - } - - Map upgradePacks = ambariMetaInfo.getUpgradePacks(stack.getName(), stack.getVersion()); - for (String key : upgradePacks.keySet()) { - UpgradePack upgradePack = upgradePacks.get(key); - final StackId sourceStack = new StackId(stack); - - final RepositoryVersionEntity rve = new RepositoryVersionEntity() {{ - setStack(new StackEntity(){{ - setStackName(sourceStack.getStackName()); - setStackVersion(sourceStack.getStackVersion()); - }}); - }}; - - final UpgradeEntity upgradeEntity = new UpgradeEntity(); - - UpgradeHistoryEntity upgradeHistoryEntity = new UpgradeHistoryEntity(){{ - setServiceName("TEST"); - setComponentName("TEST"); - setFromRepositoryVersion(rve); - setUpgrade(upgradeEntity); - }}; - - upgradeEntity.setDirection(Direction.UPGRADE); - upgradeEntity.addHistory(upgradeHistoryEntity); - upgradeEntity.setRepositoryVersion(rve); - - UpgradeContext cx = upgradeContextFactory.create(cluster, upgradeEntity); - - cx.setUpgradePack(upgradePack); - - ConfigUpgradePack configUpgradePack = ConfigurationPackBuilder.build(cx); - - // do configure tasks in the group section - List groups = upgradePack.getGroups(Direction.UPGRADE); - for( Grouping group : groups ){ - if( group instanceof ClusterGrouping ){ - ClusterGrouping clusterGrouping = (ClusterGrouping)group; - if( null != clusterGrouping.executionStages ){ - for( ExecuteStage executionStage : clusterGrouping.executionStages ){ - if( executionStage.task.getType() == Type.CONFIGURE ){ - ConfigureTask configureTask = (ConfigureTask) executionStage.task; - assertIdDefinitionExists(configureTask.id, configUpgradePack, upgradePack, - sourceStack); - - if (StringUtils.isNotBlank(executionStage.service)) { - Assert.assertEquals(executionStage.service, configureTask.associatedService); - } else { - Assert.assertTrue(null == configureTask.associatedService); - } - } - } - } - } - } - - // do tasks in the "processing" section - Map> tasks = upgradePack.getTasks(); - for( Map value : tasks.values() ){ - for( ProcessingComponent processingComponent : value.values() ){ - if (null != processingComponent.preTasks) { - for (Task preTask : processingComponent.preTasks) { - if (preTask.getType() == Type.CONFIGURE) { - ConfigureTask configureTask = (ConfigureTask) preTask; - assertIdDefinitionExists(configureTask.id, configUpgradePack, upgradePack, - sourceStack); - Assert.assertTrue(StringUtils.isNotBlank(configureTask.associatedService)); - } - } - - if (null != processingComponent.tasks) { - for (Task task : processingComponent.tasks) { - if (task.getType() == Type.CONFIGURE) { - ConfigureTask configureTask = (ConfigureTask) task; - assertIdDefinitionExists(configureTask.id, configUpgradePack, upgradePack, - sourceStack); - Assert.assertTrue(StringUtils.isNotBlank(configureTask.associatedService)); - } - } - } - - if (null != processingComponent.postTasks) { - for (Task postTask : processingComponent.postTasks) { - if (postTask.getType() == Type.CONFIGURE) { - ConfigureTask configureTask = (ConfigureTask) postTask; - assertIdDefinitionExists(configureTask.id, configUpgradePack, upgradePack, - sourceStack); - Assert.assertTrue(StringUtils.isNotBlank(configureTask.associatedService)); - } - } - } - } - } - } - } - } - - // make sure we actually checked a bunch of configs :) - Assert.assertTrue(validatedConfigCount > 100); - } - - @Test - @SuppressWarnings("unchecked") - public void testValidateConfigUpgradePacks() throws Exception { - IOFileFilter filter = new IOFileFilter() { - @Override - public boolean accept(File dir, String name) { - return false; - } - - @Override - public boolean accept(File file) { - // file has the folder named 'upgrades', ends with '.xml' and is NOT - // 'config-upgrade.xml' - if (file.getAbsolutePath().contains("upgrades") - && file.getAbsolutePath().endsWith("config-upgrade.xml")) { - return true; - } - - return false; - } - }; - - List files = new ArrayList<>(); - - files.addAll(FileUtils.listFiles(new File("src/main/resources/stacks"), filter, - FileFilterUtils.directoryFileFilter())); - - files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks"), filter, - FileFilterUtils.directoryFileFilter())); - - files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks_with_upgrade_cycle"), - filter, FileFilterUtils.directoryFileFilter())); - - ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller(); - - int filesTestedCount = 0; - for (File file : files) { - String fileContent = FileUtils.readFileToString(file, "UTF-8"); - - // these things must be in upgrade packs for them to work anyway - if (fileContent.contains(" 5); - } - - /** - * Asserts that an ID exists in a {@link ConfigUpgradePack}, throwing an - * informative message if it does not. - * - * @param id - * @param configUpgradePack - * @param upgradePack - * @param sourceStackId - */ - private void assertIdDefinitionExists(String id, ConfigUpgradePack configUpgradePack, - UpgradePack upgradePack, StackId sourceStackId) { - Assert.assertNotNull(id); - - if (configUpgradePack.enumerateConfigChangesByID().containsKey(id)) { - validatedConfigCount++; - - LOG.info("Validated {} from upgrade pack {} for {}", id, upgradePack.getTargetStack(), - sourceStackId); - - return; - } - - Assert.fail(String.format("Missing %s in upgrade from %s to %s (%s)", id, sourceStackId, - upgradePack.getTargetStack(), upgradePack.getType())); - } -} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackParsingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackParsingTest.java deleted file mode 100644 index 6113b5337c7..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/UpgradePackParsingTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.state.stack; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.ambari.server.stack.ModuleFileUnmarshaller; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Tests parsing upgrade packs in the entire codebase. This does not use Guice - * initialization, which will parse upgrade packs itself. - */ -@Category({ category.StackUpgradeTest.class}) -public class UpgradePackParsingTest { - - @Test - @SuppressWarnings("unchecked") - public void findAndValidateUpgradePacks() throws Exception { - - IOFileFilter filter = new IOFileFilter() { - @Override - public boolean accept(File dir, String name) { - return false; - } - - @Override - public boolean accept(File file) { - // file has the folder named 'upgrades', ends with '.xml' and is NOT 'config-upgrade.xml' - if (file.getAbsolutePath().contains("upgrades") && - file.getAbsolutePath().endsWith(".xml") && - !file.getAbsolutePath().contains("config-upgrade.xml")) { - - return true; - } - - return false; - } - }; - - List files = new ArrayList<>(); - - files.addAll(FileUtils.listFiles(new File("src/main/resources/stacks"), filter, - FileFilterUtils.directoryFileFilter())); - - files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks"), filter, - FileFilterUtils.directoryFileFilter())); - - files.addAll(FileUtils.listFiles(new File("src/test/resources/stacks_with_upgrade_cycle"), filter, - FileFilterUtils.directoryFileFilter())); - - ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller(); - - for (File file : files) { - String fileContent = FileUtils.readFileToString(file, "UTF-8"); - - // these things must be in upgrade packs for them to work anyway - if (fileContent.contains(" upgrades = ambariMetaInfo.getUpgradePacks("foo", "bar"); - assertTrue(upgrades.isEmpty()); - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_test")); - } - - @Test - public void testUpgradeParsing() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_test")); - UpgradePack upgrade = upgrades.get("upgrade_test"); - assertEquals("2.2.*.*", upgrade.getTarget()); - - Map> expectedStages = new LinkedHashMap>() {{ - put("ZOOKEEPER", Arrays.asList("ZOOKEEPER_SERVER")); - put("HDFS", Arrays.asList("NAMENODE", "DATANODE")); - }}; - - // !!! test the tasks - int i = 0; - for (Entry> entry : expectedStages.entrySet()) { - assertTrue(upgrade.getTasks().containsKey(entry.getKey())); - assertEquals(i++, indexOf(upgrade.getTasks(), entry.getKey())); - - // check that the number of components matches - assertEquals(entry.getValue().size(), upgrade.getTasks().get(entry.getKey()).size()); - - // check component ordering - int j = 0; - for (String comp : entry.getValue()) { - assertEquals(j++, indexOf(upgrade.getTasks().get(entry.getKey()), comp)); - } - } - - // !!! test specific tasks - assertTrue(upgrade.getTasks().containsKey("HDFS")); - assertTrue(upgrade.getTasks().get("HDFS").containsKey("NAMENODE")); - - ProcessingComponent pc = upgrade.getTasks().get("HDFS").get("NAMENODE"); - assertNotNull(pc.preTasks); - assertNotNull(pc.postTasks); - assertNotNull(pc.tasks); - assertNotNull(pc.preDowngradeTasks); - assertNotNull(pc.postDowngradeTasks); - assertEquals(1, pc.tasks.size()); - - assertEquals(3, pc.preDowngradeTasks.size()); - assertEquals(1, pc.postDowngradeTasks.size()); - - assertEquals(Task.Type.RESTART, pc.tasks.get(0).getType()); - assertEquals(RestartTask.class, pc.tasks.get(0).getClass()); - - - assertTrue(upgrade.getTasks().containsKey("ZOOKEEPER")); - assertTrue(upgrade.getTasks().get("ZOOKEEPER").containsKey("ZOOKEEPER_SERVER")); - - pc = upgrade.getTasks().get("HDFS").get("DATANODE"); - assertNotNull(pc.preDowngradeTasks); - assertEquals(0, pc.preDowngradeTasks.size()); - assertNotNull(pc.postDowngradeTasks); - assertEquals(1, pc.postDowngradeTasks.size()); - - - pc = upgrade.getTasks().get("ZOOKEEPER").get("ZOOKEEPER_SERVER"); - assertNotNull(pc.preTasks); - assertEquals(1, pc.preTasks.size()); - assertNotNull(pc.postTasks); - assertEquals(1, pc.postTasks.size()); - assertNotNull(pc.tasks); - assertEquals(1, pc.tasks.size()); - - pc = upgrade.getTasks().get("YARN").get("NODEMANAGER"); - assertNotNull(pc.preTasks); - assertEquals(2, pc.preTasks.size()); - Task t = pc.preTasks.get(1); - assertEquals(ConfigureTask.class, t.getClass()); - ConfigureTask ct = (ConfigureTask) t; - // check that the Configure task successfully parsed id - assertEquals("hdp_2_1_1_nm_pre_upgrade", ct.getId()); - assertFalse(ct.supportsPatch); - } - - @Test - public void testGroupOrdersForRolling() { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_test_checks")); - UpgradePack upgrade = upgrades.get("upgrade_test_checks"); - - PrerequisiteCheckConfig prerequisiteCheckConfig = upgrade.getPrerequisiteCheckConfig(); - assertNotNull(prerequisiteCheckConfig); - assertNotNull(prerequisiteCheckConfig.globalProperties); - assertTrue(prerequisiteCheckConfig.getGlobalProperties().containsKey("global-property-1")); - assertEquals("global-value-1", prerequisiteCheckConfig.getGlobalProperties().get("global-property-1")); - assertNotNull(prerequisiteCheckConfig.prerequisiteCheckProperties); - assertEquals(2, prerequisiteCheckConfig.prerequisiteCheckProperties.size()); - assertNotNull(prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck")); - assertTrue(prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck").containsKey("dfs-protocols-regex")); - assertEquals("^([^:]*dfs|wasb|ecs):.*", prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck").get("dfs-protocols-regex")); - assertNotNull(prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck")); - assertTrue(prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck").containsKey("dfs-protocols-regex")); - assertEquals("^([^:]*dfs|wasb|ecs):.*", prerequisiteCheckConfig.getCheckProperties( - "org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck").get("dfs-protocols-regex")); - - - List expected_up = Arrays.asList( - "PRE_CLUSTER", - "ZOOKEEPER", - "CORE_MASTER", - "SERVICE_CHECK_1", - "CORE_SLAVES", - "SERVICE_CHECK_2", - "OOZIE", - "POST_CLUSTER"); - - List expected_down = Arrays.asList( - "PRE_CLUSTER", - "OOZIE", - "CORE_SLAVES", - "SERVICE_CHECK_2", - "CORE_MASTER", - "SERVICE_CHECK_1", - "ZOOKEEPER", - "POST_CLUSTER"); - - Grouping serviceCheckGroup = null; - - int i = 0; - List groups = upgrade.getGroups(Direction.UPGRADE); - for (Grouping g : groups) { - assertEquals(expected_up.get(i), g.name); - i++; - - if (g.name.equals("SERVICE_CHECK_1")) { - serviceCheckGroup = g; - } - } - - List expected_priority = Arrays.asList("HDFS", "HBASE", "YARN"); - - assertNotNull(serviceCheckGroup); - assertEquals(ServiceCheckGrouping.class, serviceCheckGroup.getClass()); - ServiceCheckGrouping scg = (ServiceCheckGrouping) serviceCheckGroup; - - Set priorities = scg.getPriorities(); - assertEquals(3, priorities.size()); - - i = 0; - for (String s : priorities) { - assertEquals(expected_priority.get(i++), s); - } - - - i = 0; - groups = upgrade.getGroups(Direction.DOWNGRADE); - for (Grouping g : groups) { - assertEquals(expected_down.get(i), g.name); - i++; - } - - } - - - // TODO AMBARI-12698, add the Downgrade case - @Test - public void testGroupOrdersForNonRolling() { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_test_nonrolling")); - UpgradePack upgrade = upgrades.get("upgrade_test_nonrolling"); - - List expected_up = Arrays.asList( - "PRE_CLUSTER", - "Stop High-Level Daemons", - "Backups", - "Stop Low-Level Daemons", - "UPDATE_DESIRED_REPOSITORY_ID", - "ALL_HOST_OPS", - "ZOOKEEPER", - "HDFS", - "MR and YARN", - "POST_CLUSTER"); - - List expected_down = Arrays.asList( - "Restore Backups", - "UPDATE_DESIRED_REPOSITORY_ID", - "ALL_HOST_OPS", - "ZOOKEEPER", - "HDFS", - "MR and YARN", - "POST_CLUSTER"); - - - Iterator itr_up = expected_up.iterator(); - List upgrade_groups = upgrade.getGroups(Direction.UPGRADE); - for (Grouping g : upgrade_groups) { - assertEquals(true, itr_up.hasNext()); - assertEquals(itr_up.next(), g.name); - } - - Iterator itr_down = expected_down.iterator(); - List downgrade_groups = upgrade.getGroups(Direction.DOWNGRADE); - for (Grouping g : downgrade_groups) { - assertEquals(true, itr_down.hasNext()); - assertEquals(itr_down.next(), g.name); - } - } - - - @Test - public void testDirectionForRolling() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_direction")); - UpgradePack upgrade = upgrades.get("upgrade_direction"); - assertTrue(upgrade.getType() == UpgradeType.ROLLING); - - List groups = upgrade.getGroups(Direction.UPGRADE); - assertEquals(4, groups.size()); - Grouping group = groups.get(2); - assertEquals(ClusterGrouping.class, group.getClass()); - ClusterGrouping cluster_group = (ClusterGrouping) group; - assertEquals("Run on All", group.title); - - cluster_group = (ClusterGrouping) groups.get(3); - List stages = cluster_group.executionStages; - assertEquals(3, stages.size()); - assertNotNull(stages.get(0).intendedDirection); - assertEquals(Direction.DOWNGRADE, stages.get(0).intendedDirection); - - groups = upgrade.getGroups(Direction.DOWNGRADE); - assertEquals(3, groups.size()); - // there are two clustergroupings at the end - group = groups.get(1); - assertEquals(ClusterGrouping.class, group.getClass()); - assertEquals("Run on All", group.title); - - group = groups.get(2); - assertEquals(ClusterGrouping.class, group.getClass()); - assertEquals("Finalize Upgrade", group.title); - } - @Test - public void testSkippableFailures() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - Set keys = upgrades.keySet(); - for (String key : keys) { - Assert.assertFalse(upgrades.get(key).isComponentFailureAutoSkipped()); - Assert.assertFalse(upgrades.get(key).isServiceCheckFailureAutoSkipped()); - } - - upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - UpgradePack upgradePack = upgrades.get("upgrade_test_skip_failures"); - Assert.assertTrue(upgradePack.isComponentFailureAutoSkipped()); - Assert.assertTrue(upgradePack.isServiceCheckFailureAutoSkipped()); - } - - /** - * Tests that the XML for not auto skipping skippable failures works. - * - * @throws Exception - */ - @Test - public void testNoAutoSkipFailure() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - UpgradePack upgradePack = upgrades.get("upgrade_test_skip_failures"); - - List groups = upgradePack.getGroups(Direction.UPGRADE); - for (Grouping grouping : groups) { - if (grouping.name.equals("SKIPPABLE_BUT_NOT_AUTO_SKIPPABLE")) { - Assert.assertFalse(grouping.supportsAutoSkipOnFailure); - } else { - Assert.assertTrue(grouping.supportsAutoSkipOnFailure); - } - } - } - - @Test - public void testDirectionForNonRolling() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - assertTrue(upgrades.size() > 0); - assertTrue(upgrades.containsKey("upgrade_test_nonrolling")); - UpgradePack upgrade = upgrades.get("upgrade_test_nonrolling"); - assertTrue(upgrade.getType() == UpgradeType.NON_ROLLING); - - List groups = upgrade.getGroups(Direction.UPGRADE); - assertEquals(10, groups.size()); - - Grouping group = null; - ClusterGrouping clusterGroup = null; - UpdateStackGrouping updateStackGroup = null; - StopGrouping stopGroup = null; - RestartGrouping restartGroup = null; + public void testLifecycleOrdering() throws Exception { + Collection typeSet = LifecycleType.ordered(); - group = groups.get(0); - assertEquals(ClusterGrouping.class, group.getClass()); - clusterGroup = (ClusterGrouping) group; - assertEquals("Prepare Upgrade", clusterGroup.title); - assertNull(clusterGroup.parallelScheduler); + LifecycleType[] types = typeSet.toArray(new LifecycleType[typeSet.size()]); - group = groups.get(1); - assertEquals(StopGrouping.class, group.getClass()); - stopGroup = (StopGrouping) group; - assertEquals("Stop Daemons for High-Level Services", stopGroup.title); - assertNotNull(stopGroup.parallelScheduler); - assertEquals(ParallelScheduler.DEFAULT_MAX_DEGREE_OF_PARALLELISM, stopGroup.parallelScheduler.maxDegreeOfParallelism); - - group = groups.get(2); - assertEquals(ClusterGrouping.class, group.getClass()); - clusterGroup = (ClusterGrouping) group; - assertEquals("Take Backups", clusterGroup.title); - assertNull(clusterGroup.parallelScheduler); - - group = groups.get(3); - assertEquals(StopGrouping.class, group.getClass()); - stopGroup = (StopGrouping) group; - assertEquals("Stop Daemons for Low-Level Services", stopGroup.title); - assertNotNull(stopGroup.parallelScheduler); - assertEquals(ParallelScheduler.DEFAULT_MAX_DEGREE_OF_PARALLELISM, stopGroup.parallelScheduler.maxDegreeOfParallelism); - - group = groups.get(4); - assertEquals(UpdateStackGrouping.class, group.getClass()); - updateStackGroup = (UpdateStackGrouping) group; - assertEquals("Update Desired Stack Id", updateStackGroup.title); - assertNull(updateStackGroup.parallelScheduler); - - group = groups.get(5); - assertEquals(ClusterGrouping.class, group.getClass()); - clusterGroup = (ClusterGrouping) group; - assertEquals("Set Version On All Hosts", clusterGroup.title); - assertNull(clusterGroup.parallelScheduler); - - group = groups.get(6); - assertEquals(RestartGrouping.class, group.getClass()); - restartGroup = (RestartGrouping) group; - assertEquals("Zookeeper", restartGroup.title); - assertNull(restartGroup.parallelScheduler); - - group = groups.get(7); - assertEquals(RestartGrouping.class, group.getClass()); - restartGroup = (RestartGrouping) group; - assertEquals("HDFS", restartGroup.title); - assertNotNull(restartGroup.parallelScheduler); - assertEquals(2, restartGroup.parallelScheduler.maxDegreeOfParallelism); - - group = groups.get(8); - assertEquals(RestartGrouping.class, group.getClass()); - restartGroup = (RestartGrouping) group; - assertEquals("MR and YARN", restartGroup.title); - assertNotNull(restartGroup.parallelScheduler); - assertEquals(ParallelScheduler.DEFAULT_MAX_DEGREE_OF_PARALLELISM, restartGroup.parallelScheduler.maxDegreeOfParallelism); - - group = groups.get(9); - assertEquals(ClusterGrouping.class, group.getClass()); - clusterGroup = (ClusterGrouping) group; - assertEquals("Finalize {{direction.text.proper}}", clusterGroup.title); - assertNull(clusterGroup.parallelScheduler); + assertEquals(LifecycleType.INSTALL, types[0]); + assertEquals(LifecycleType.FINALIZE, types[types.length-1]); } - - /** - * Tests that the service level XML merges correctly for 2.0.5/HDFS/HDP/2.2.0. - * - * @throws Exception - */ @Test - public void testServiceLevelUpgradePackMerge() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - assertTrue(upgrades.containsKey("upgrade_test_15388")); - - UpgradePack upgradePack = upgrades.get("upgrade_test_15388"); - - List checks = upgradePack.getPrerequisiteChecks(); - assertEquals(11, checks.size()); - assertTrue(checks.contains("org.apache.ambari.server.checks.FooCheck")); - - List groups = upgradePack.getGroups(Direction.UPGRADE); - assertEquals(8, groups.size()); - Grouping group = groups.get(0); - assertEquals(ClusterGrouping.class, group.getClass()); - ClusterGrouping cluster_group = (ClusterGrouping) group; - assertEquals("Pre {{direction.text.proper}}", cluster_group.title); - - List stages = cluster_group.executionStages; - assertEquals(5, stages.size()); - ExecuteStage stage = stages.get(3); - assertEquals("Backup FOO", stage.title); - - group = groups.get(2); - assertEquals("Core Masters", group.title); - List services = group.services; - assertEquals(3, services.size()); - UpgradePack.OrderService service = services.get(2); - assertEquals("HBASE", service.serviceName); - - group = groups.get(3); - assertEquals("Core Slaves", group.title); - services = group.services; - assertEquals(3, services.size()); - service = services.get(1); - assertEquals("HBASE", service.serviceName); - - group = groups.get(4); - assertEquals(ServiceCheckGrouping.class, group.getClass()); - ServiceCheckGrouping scGroup = (ServiceCheckGrouping) group; - Set priorityServices = scGroup.getPriorities(); - assertEquals(4, priorityServices.size()); - Iterator serviceIterator = priorityServices.iterator(); - assertEquals("ZOOKEEPER", serviceIterator.next()); - assertEquals("HBASE", serviceIterator.next()); - - group = groups.get(5); - assertEquals("Hive", group.title); - - group = groups.get(6); - assertEquals("Foo", group.title); - services = group.services; - assertEquals(2, services.size()); - service = services.get(1); - assertEquals("FOO2", service.serviceName); - - Map> tasks = upgradePack.getTasks(); - assertTrue(tasks.containsKey("HBASE")); - - // !!! generalized upgrade pack shouldn't be in this - boolean found = false; - for (Grouping grouping : upgradePack.getAllGroups()) { - if (grouping.name.equals("GANGLIA_UPGRADE")) { - found = true; - break; - } - } - assertFalse(found); - - // !!! test merge of a generalized upgrade pack - upgradePack = upgrades.get("upgrade_test_conditions"); - assertNotNull(upgradePack); - for (Grouping grouping : upgradePack.getAllGroups()) { - if (grouping.name.equals("GANGLIA_UPGRADE")) { - found = true; - break; - } - } - assertTrue(found); - } - - - @Test - public void testPackWithHostGroup() { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.2.0"); - UpgradePack upgradePack = upgrades.get("upgrade_test_host_ordered"); - - assertNotNull(upgradePack); - assertEquals(upgradePack.getType(), UpgradeType.HOST_ORDERED); - assertEquals(3, upgradePack.getAllGroups().size()); - - assertEquals(HostOrderGrouping.class, upgradePack.getAllGroups().get(0).getClass()); - assertEquals(Grouping.class, upgradePack.getAllGroups().get(1).getClass()); - } - - @Test - public void testDowngradeComponentTasks() throws Exception { - Map upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1"); - UpgradePack upgradePack = upgrades.get("upgrade_component_tasks_test"); - assertNotNull(upgradePack); - - Map> components = upgradePack.getTasks(); - assertTrue(components.containsKey("ZOOKEEPER")); - assertTrue(components.containsKey("HDFS")); - - Map zkMap = components.get("ZOOKEEPER"); - Map hdfsMap = components.get("HDFS"); - - assertTrue(zkMap.containsKey("ZOOKEEPER_SERVER")); - assertTrue(zkMap.containsKey("ZOOKEEPER_CLIENT")); - assertTrue(hdfsMap.containsKey("NAMENODE")); - assertTrue(hdfsMap.containsKey("DATANODE")); - assertTrue(hdfsMap.containsKey("HDFS_CLIENT")); - assertTrue(hdfsMap.containsKey("JOURNALNODE")); - - ProcessingComponent zkServer = zkMap.get("ZOOKEEPER_SERVER"); - ProcessingComponent zkClient = zkMap.get("ZOOKEEPER_CLIENT"); - ProcessingComponent hdfsNN = hdfsMap.get("NAMENODE"); - ProcessingComponent hdfsDN = hdfsMap.get("DATANODE"); - ProcessingComponent hdfsClient = hdfsMap.get("HDFS_CLIENT"); - ProcessingComponent hdfsJN = hdfsMap.get("JOURNALNODE"); - - // ZK server has only pretasks defined, with pre-downgrade being a copy of pre-upgrade - assertNotNull(zkServer.preTasks); - assertNotNull(zkServer.preDowngradeTasks); - assertNull(zkServer.postTasks); - assertNull(zkServer.postDowngradeTasks); - assertEquals(1, zkServer.preTasks.size()); - assertEquals(1, zkServer.preDowngradeTasks.size()); - - // ZK client has only post-tasks defined, with post-downgrade being a copy of pre-upgrade - assertNull(zkClient.preTasks); - assertNull(zkClient.preDowngradeTasks); - assertNotNull(zkClient.postTasks); - assertNotNull(zkClient.postDowngradeTasks); - assertEquals(1, zkClient.postTasks.size()); - assertEquals(1, zkClient.postDowngradeTasks.size()); - - // NN has only pre-tasks defined, with an empty pre-downgrade - assertNotNull(hdfsNN.preTasks); - assertNotNull(hdfsNN.preDowngradeTasks); - assertNull(hdfsNN.postTasks); - assertNull(hdfsNN.postDowngradeTasks); - assertEquals(1, hdfsNN.preTasks.size()); - assertEquals(0, hdfsNN.preDowngradeTasks.size()); - - // DN has only post-tasks defined, with post-downgrade being empty - assertNull(hdfsDN.preTasks); - assertNull(hdfsDN.preDowngradeTasks); - assertNotNull(hdfsDN.postTasks); - assertNotNull(hdfsDN.postDowngradeTasks); - assertEquals(1, hdfsDN.postTasks.size()); - assertEquals(0, hdfsDN.postDowngradeTasks.size()); - - // HDFS client has only post and post-downgrade tasks - assertNull(hdfsClient.preTasks); - assertNotNull(hdfsClient.preDowngradeTasks); - assertNull(hdfsClient.postTasks); - assertNotNull(hdfsClient.postDowngradeTasks); - assertEquals(1, hdfsClient.preDowngradeTasks.size()); - assertEquals(1, hdfsClient.postDowngradeTasks.size()); + public void testUpgradeParsing() throws Exception { + File f = new File("src/test/resources/mpacks-v2/upgrade-packs/upgrade-basic.xml"); - // JN has differing tasks for pre and post downgrade - assertNotNull(hdfsJN.preTasks); - assertNotNull(hdfsJN.preDowngradeTasks); - assertNotNull(hdfsJN.postTasks); - assertNotNull(hdfsJN.postDowngradeTasks); - assertEquals(1, hdfsJN.preTasks.size()); - assertEquals(2, hdfsJN.preDowngradeTasks.size()); - assertEquals(1, hdfsJN.postTasks.size()); - assertEquals(2, hdfsJN.postDowngradeTasks.size()); + ModuleFileUnmarshaller unmarshaller = new ModuleFileUnmarshaller(); - // make sure all ids are accounted for + UpgradePack upgradepack = unmarshaller.unmarshal(UpgradePack.class, f, true); - Set allIds = Sets.newHashSet("some_id", "some_id1", "some_id2", "some_id3", "some_id4", "some_id5"); + assertEquals(UpgradeType.ROLLING, upgradepack.getType()); + assertEquals(6, upgradepack.lifecycles.size()); - @SuppressWarnings("unchecked") - Set> allTasks = Sets.newHashSet(hdfsJN.preTasks, hdfsJN.preDowngradeTasks, - hdfsJN.postTasks, hdfsJN.postDowngradeTasks); + Optional upgradeLifecycle = upgradepack.lifecycles.stream() + .filter(l -> l.type == LifecycleType.UPGRADE).findFirst(); - for (List list : allTasks) { - for (Task t : list) { - assertEquals(ConfigureTask.class, t.getClass()); + Optional startLifecycle = upgradepack.lifecycles.stream() + .filter(l -> l.type == LifecycleType.START).findFirst(); - ConfigureTask ct = (ConfigureTask) t; - assertTrue(allIds.contains(ct.id)); + assertTrue(upgradeLifecycle.isPresent()); + assertFalse(startLifecycle.isPresent()); - allIds.remove(ct.id); - } - } + List groups = upgradeLifecycle.get().groups; - assertTrue(allIds.isEmpty()); + assertEquals(29, groups.size()); } - - private int indexOf(Map map, String keyToFind) { - int result = -1; - - int i = 0; - for (Entry entry : map.entrySet()) { - if (entry.getKey().equals(keyToFind)) { - return i; - } - i++; - } - - return result; - } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java index 7c6c258e576..7a17ccc5025 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostTest.java @@ -133,7 +133,7 @@ public void setup() throws Exception { createCluster(stackId, clusterName, repositoryVersion.getVersion()); - serviceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "test_group", new HashSet<>()); + serviceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "test_group", stackId, new HashSet<>()); hostAttributes.put("os_family", "redhat"); hostAttributes.put("os_release_version", "5.9"); Set hostNames = new HashSet<>(); @@ -713,7 +713,7 @@ public void testStaleConfigs() throws Exception { String clusterName = "c2"; createCluster(stackId, clusterName, ""); - ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", new HashSet<>()); + ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", stackId, new HashSet<>()); final String hostName = "h3"; Set hostNames = new HashSet<>(); @@ -906,7 +906,7 @@ public void testStaleConfigsAttributes() throws Exception { String clusterName = "c2"; createCluster(stackId, clusterName, ""); - ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", new HashSet<>()); + ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", stackId, new HashSet<>()); final String hostName = "h3"; Set hostNames = new HashSet<>(); @@ -1035,7 +1035,7 @@ public void testMaintenance() throws Exception { String clusterName = "c2"; createCluster(stackId, clusterName, ""); - ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", new HashSet<>()); + ServiceGroup customServiceGroup = serviceGroupFactory.createNew(clusters.getCluster(clusterName), "custom_group", stackId, new HashSet<>()); final String hostName = "h3"; Set hostNames = new HashSet<>(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/testing/DBInconsistencyTests.java b/ambari-server/src/test/java/org/apache/ambari/server/testing/DBInconsistencyTests.java index 2ba9c39df62..c78b0ee2e09 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/testing/DBInconsistencyTests.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/testing/DBInconsistencyTests.java @@ -91,7 +91,7 @@ public void testOrphanedSCHDesiredEntityReAdd() throws Exception { Assert.assertNotNull(clusterId); Cluster cluster = clusters.getCluster(OrmTestHelper.CLUSTER_NAME); - ServiceGroup serviceGroup = cluster.addServiceGroup("CORE"); + ServiceGroup serviceGroup = cluster.addServiceGroup("CORE", "HDP-1.0"); Assert.assertNotNull(cluster); helper.addHost(clusters, cluster, "h1"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index b56835424eb..895562d7756 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -336,7 +336,7 @@ public void testCreateAmbariResources() throws Exception { assertEquals(String.format("%s-%s", STACK_NAME, STACK_VERSION), clusterRequest.getStackVersion()); Set serviceGroupRequests = serviceGroupRequestCapture.getValue(); - Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), AmbariContext.DEFAULT_SERVICE_GROUP_NAME)); + Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), AmbariContext.DEFAULT_SERVICE_GROUP_NAME, clusterRequest.getStackVersion())); assertEquals(expectedServiceGroupRequests, serviceGroupRequests); Collection serviceRequests = serviceRequestCapture.getValue(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestVersionUtils.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestVersionUtils.java index 08c0a06e498..ebb7278bad5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/TestVersionUtils.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/TestVersionUtils.java @@ -277,17 +277,41 @@ public void testMpackVersionWithValidVersions() { Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h0-b10").compareTo(MpackVersion.parse("2.2.3-b10"))); Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h0-b10").compareTo(MpackVersion.parse("1.2.3-h1-b10"))); Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h0-b10").compareTo(MpackVersion.parse("1.2.3-h0-b11"))); + + Assert.assertEquals(0, MpackVersion.parse("1", false).compareTo(MpackVersion.parse("1.0.0-h0-b0"))); + Assert.assertEquals(0, MpackVersion.parse("1.0", false).compareTo(MpackVersion.parse("1.0.0-h0-b0"))); + Assert.assertEquals(0, MpackVersion.parse("1.0.0", false).compareTo(MpackVersion.parse("1.0.0-h0-b0"))); + Assert.assertEquals(0, MpackVersion.parse("1.0.0-b0", false).compareTo(MpackVersion.parse("1.0.0-h0-b0"))); + Assert.assertEquals(0, MpackVersion.parse("1.0.0-h0-b0", false).compareTo(MpackVersion.parse("1.0.0-h0-b0"))); + + Assert.assertEquals(-1, MpackVersion.parse("1", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(-1, MpackVersion.parse("1.0", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(-1, MpackVersion.parse("1.0.0", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(-1, MpackVersion.parse("1.0.0-b0", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(-1, MpackVersion.parse("1.0.0-h0-b0", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + + Assert.assertEquals(1, MpackVersion.parse("2", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(1, MpackVersion.parse("1.1", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); + Assert.assertEquals(1, MpackVersion.parse("1.0.1", false).compareTo(MpackVersion.parse("1.0.0-h0-b111"))); } @Test public void testStackVersionWithValidVersions() { Assert.assertEquals(1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); - Assert.assertEquals(1, MpackVersion.parseStackVersion("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); + Assert.assertEquals(1, MpackVersion.parse("1.2.4-h1-b1").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); + Assert.assertEquals(1, MpackVersion.parse("1.3.3-h1-b1").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); + Assert.assertEquals(1, MpackVersion.parse("2.2.3-h1-b1").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); + + Assert.assertEquals(0, MpackVersion.parse("1.2.3-h4-b888").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); + Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.3.11-888"))); + Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.4.1-1"))); + Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.3.3.1-1"))); + Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("2.2.3.1-1"))); + Assert.assertEquals(1, MpackVersion.parseStackVersion("1.2.3.4-999").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); - Assert.assertEquals(0, MpackVersion.parseStackVersion("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.3-h10-b10"))); - Assert.assertEquals(0, MpackVersion.parseStackVersion("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("1.2.3.10-10"))); - Assert.assertEquals(-1, MpackVersion.parse("1.2.3-h10-b10").compareTo(MpackVersion.parseStackVersion("2.2.3.4-888"))); + Assert.assertEquals(0, MpackVersion.parseStackVersion("1.2.3.4-999").compareTo(MpackVersion.parseStackVersion("1.2.3.4-999"))); + Assert.assertEquals(-1, MpackVersion.parseStackVersion("1.2.3.1-999").compareTo(MpackVersion.parseStackVersion("1.2.3.4-888"))); } @Test diff --git a/ambari-server/src/test/python/TestSettings.py b/ambari-server/src/test/python/TestSettings.py index e117747033c..f8761f772c0 100644 --- a/ambari-server/src/test/python/TestSettings.py +++ b/ambari-server/src/test/python/TestSettings.py @@ -51,26 +51,35 @@ def test_entries_for_supported_type(self): Script.config = TestSettings._get_simple_command() # For stackSettings - self.assertEquals(Script.config['stackSettings'], settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, None)) - self.assertEquals(Script.config['stackSettings'], settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE)) + stackSettings = dict((k, settings.convert_value(v)) for k, v in Script.config['stackSettings'].items()) + self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, None)) + self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE)) # For clusterSettings - self.assertEquals(Script.config['clusterSettings'], settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE, None)) - self.assertEquals(Script.config['clusterSettings'], settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE)) + clusterSettings = dict((k, settings.convert_value(v)) for k, v in Script.config['clusterSettings'].items()) + self.assertEquals(clusterSettings, settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE, None)) + self.assertEquals(clusterSettings, settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE)) + + + def test_boolean_values_are_converted(self): + Script.config = TestSettings._get_simple_command() + + self.assertEquals(settings.get_setting_value(settings.CLUSTER_SETTINGS_TYPE, 'manage_dirs_on_root'), True) + self.assertEquals(settings.get_setting_value(settings.CLUSTER_SETTINGS_TYPE, 'recovery_enabled'), False) def test_full_subset_of_entries_for_supported_type(self): Script.config = TestSettings._get_simple_command() # For stackSettings - stackSettings = Script.config['stackSettings'] + stackSettings = dict((k, settings.convert_value(v)) for k, v in Script.config['stackSettings'].items()) self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, set(stackSettings.keys()))) self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, frozenset(stackSettings.keys()))) self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, tuple(stackSettings.keys()))) self.assertEquals(stackSettings, settings.get_setting_type_entries(settings.STACK_SETTINGS_TYPE, list(stackSettings.keys()))) # For clusterSettings - clusterSettings = Script.config['clusterSettings'] + clusterSettings = dict((k, settings.convert_value(v)) for k, v in Script.config['clusterSettings'].items()) self.assertEquals(clusterSettings, settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE, set(clusterSettings.keys()))) self.assertEquals(clusterSettings, settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE, frozenset(clusterSettings.keys()))) self.assertEquals(clusterSettings, settings.get_setting_type_entries(settings.CLUSTER_SETTINGS_TYPE, tuple(clusterSettings.keys()))) @@ -155,7 +164,7 @@ def test_value_of_supported_setting_type_non_existent_name(self): # For clusterSettings self.assertTrue(settings.get_setting_value(settings.CLUSTER_SETTINGS_TYPE, 'non_existing_key') is None) - def test_value_of_supported_setting_type_empy_string_name(self): + def test_value_of_supported_setting_type_empty_string_name(self): Script.config = TestSettings._get_simple_command() # For stackSettings @@ -179,12 +188,12 @@ def test_value_of_supported_setting_type_and_name(self): # For stackSettings stack_settings = Script.config['stackSettings'] for k in stack_settings.keys(): - self.assertEquals(stack_settings[k], settings.get_setting_value(settings.STACK_SETTINGS_TYPE, k)) + self.assertEquals(settings.convert_value(stack_settings[k]), settings.get_setting_value(settings.STACK_SETTINGS_TYPE, k)) # For clusterSettings cluster_settings = Script.config['clusterSettings'] for k in cluster_settings.keys(): - self.assertEquals(cluster_settings[k], settings.get_setting_value(settings.CLUSTER_SETTINGS_TYPE, k)) + self.assertEquals(settings.convert_value(cluster_settings[k]), settings.get_setting_value(settings.CLUSTER_SETTINGS_TYPE, k)) @staticmethod @@ -199,6 +208,7 @@ def _get_simple_command(): }, "clusterSettings": { "recovery_enabled": "false", + "manage_dirs_on_root": "true", "smokeuser": "ambari-qa", "recovery_type": "AUTO_START", "user_group": "hadoop", diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py index d91bcf4606e..0b7c3a402f5 100644 --- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py +++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py @@ -553,7 +553,7 @@ def side_effect(path): } }, "direction":"UPGRADE", - "type":"nonrolling_upgrade", + "type":"express_upgrade", "isRevert":False, "orchestration":"STANDARD" } diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py index bcba1925a90..ff684760b32 100644 --- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py +++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py @@ -225,7 +225,7 @@ def test_pre_upgrade_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, } }, "direction":"UPGRADE", - "type":"nonrolling_upgrade", + "type":"express_upgrade", "isRevert":False, "orchestration":"STANDARD" } diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/test/resources/mpacks-v2/upgrade-packs/upgrade-basic.xml similarity index 55% rename from ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml rename to ambari-server/src/test/resources/mpacks-v2/upgrade-packs/upgrade-basic.xml index 4f10ea54207..1ebb00faa44 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/test/resources/mpacks-v2/upgrade-packs/upgrade-basic.xml @@ -21,7 +21,7 @@ HDP-2.6 false false - ROLLING + rolling org.apache.ambari.server.checks.HiveMultipleMetastoreCheck @@ -44,464 +44,492 @@ - - - - UPGRADE - - - - prepare_rolling_upgrade - - - - - - - prepare - - - - - - UPGRADE - true - false - - - - - take_snapshot - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. - - - - - Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. - - - - - - false - - ZOOKEEPER_SERVER - - - - - true - false - - RANGER_ADMIN - RANGER_USERSYNC - RANGER_TAGSYNC - - - - - true - false - - RANGER_KMS_SERVER - - - - - - true - - KAFKA_BROKER - - - - - - - - - - - - - false - - NAMENODE - JOURNALNODE - ZKFC - - - - HISTORYSERVER - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - - - - HBASE_MASTER - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - MAPREDUCE2 - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - false - true - - DATANODE - NFS_GATEWAY - - - - HBASE_REGIONSERVER - PHOENIX_QUERY_SERVER - - - - NODEMANAGER - - - - 20 - Verification Required - The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - - - AMBARI_METRICS - LOGSEARCH - - - - - - true - - SLIDER - - - - - true - false - - HIVE_SERVER - HIVE_METASTORE - HIVE_SERVER_INTERACTIVE - WEBHCAT_SERVER - - - - - true - false - false - - SPARK_JOBHISTORYSERVER - SPARK_THRIFTSERVER - LIVY_SERVER - - - - - true - false - false - - SPARK2_JOBHISTORYSERVER - SPARK2_THRIFTSERVER - LIVY2_SERVER - - - - - true - false - false - - ZEPPELIN_MASTER - - - - - true - - SPARK_CLIENT - - - - - true - - SPARK2_CLIENT - - - - - false - true - false - - ATLAS_SERVER - - - - - false - true - true - - ATLAS_CLIENT - - - - - true - false - false - - OOZIE_SERVER - - - - - true - - OOZIE_CLIENT - - - - - true - false - false - - FALCON_SERVER - - - - - true - - FALCON_CLIENT - - - - - false - - ZOOKEEPER_CLIENT - - - HDFS_CLIENT - - - - YARN_CLIENT - - - - MAPREDUCE2_CLIENT - - - - TEZ_CLIENT - - - - HBASE_CLIENT - - - - PIG - - - - SQOOP - - - - MAHOUT - - - - HIVE_CLIENT - HCAT - - - - - true - false - false - - DRUID_HISTORICAL - DRUID_MIDDLEMANAGER - DRUID_BROKER - DRUID_ROUTER - DRUID_COORDINATOR - DRUID_OVERLORD - DRUID_SUPERSET - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - HBASE - DRUID - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - KNOX_GATEWAY - - - - - true - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - true - - FLUME_HANDLER - - - - - true - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - COMPLETE - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - + + + + + + + + + + + + + + + + UPGRADE - - Please confirm you are ready to finalize. - - - - + + + + prepare_rolling_upgrade + + + + + + + prepare + + + + + + + + + UPGRADE - - You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: - hbase> list_snapshots - Once you have found an existing snapshot which you would like to remove, you can use the following command: - hbase> delete_snapshot 'snapshotName' - - - - - - - finalize_rolling_upgrade - - - - - - - - - + true + false + + + + + take_snapshot + + + + + + Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. + + + + + + Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. + + + + + + Before continuing, please backup the Ranger Admin database on the following host(s): {{hosts.all}}. + + + + + + Before continuing, please backup Ranger KMS database on the following host(s): {{hosts.all}}. + + + + + Before continuing, please backup Zeppelin notebooks directory on the following host(s): {{hosts.all}}. + + + + + + + + + + false + + ZOOKEEPER_SERVER + + + + + true + false + + RANGER_ADMIN + RANGER_USERSYNC + RANGER_TAGSYNC + + + + + true + false + + RANGER_KMS_SERVER + + + + + + true + + KAFKA_BROKER + + + + + + + + + + + + + false + + NAMENODE + JOURNALNODE + ZKFC + + + + HISTORYSERVER + + + + APP_TIMELINE_SERVER + RESOURCEMANAGER + + + + HBASE_MASTER + + + + + UPGRADE + true + + ZOOKEEPER + HDFS + YARN + MAPREDUCE2 + HBASE + + + AMBARI_METRICS + LOGSEARCH + + + + + false + true + + DATANODE + NFS_GATEWAY + + + + HBASE_REGIONSERVER + PHOENIX_QUERY_SERVER + + + + NODEMANAGER + + + + 20 + Verification Required + The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services. + + + + + UPGRADE + true + + ZOOKEEPER + HDFS + YARN + HBASE + + + AMBARI_METRICS + LOGSEARCH + + + + + + true + + SLIDER + + + + + true + false + + HIVE_SERVER + HIVE_METASTORE + HIVE_SERVER_INTERACTIVE + WEBHCAT_SERVER + + + + + true + false + false + + SPARK_JOBHISTORYSERVER + SPARK_THRIFTSERVER + LIVY_SERVER + + + + + true + false + false + + SPARK2_JOBHISTORYSERVER + SPARK2_THRIFTSERVER + LIVY2_SERVER + + + + + true + false + false + + ZEPPELIN_MASTER + + + + + true + + SPARK_CLIENT + + + + + true + + SPARK2_CLIENT + + + + + false + true + false + + ATLAS_SERVER + + + + + false + true + true + + ATLAS_CLIENT + + + + + true + false + false + + OOZIE_SERVER + + + + + true + + OOZIE_CLIENT + + + + + true + false + false + + FALCON_SERVER + + + + + true + + FALCON_CLIENT + + + + + false + + ZOOKEEPER_CLIENT + + + HDFS_CLIENT + + + + YARN_CLIENT + + + + MAPREDUCE2_CLIENT + + + + TEZ_CLIENT + + + + HBASE_CLIENT + + + + PIG + + + + SQOOP + + + + MAHOUT + + + + HIVE_CLIENT + HCAT + + + + + true + false + false + + DRUID_HISTORICAL + DRUID_MIDDLEMANAGER + DRUID_BROKER + DRUID_ROUTER + DRUID_COORDINATOR + DRUID_OVERLORD + DRUID_SUPERSET + + + + + UPGRADE + true + + ZOOKEEPER + HDFS + YARN + HBASE + DRUID + + + AMBARI_METRICS + LOGSEARCH + + + + + true + + KNOX_GATEWAY + + + + + true + + NIMBUS + SUPERVISOR + STORM_UI_SERVER + DRPC_SERVER + + + + + true + + FLUME_HANDLER + + + + + true + + ACCUMULO_MASTER + ACCUMULO_TSERVER + ACCUMULO_MONITOR + ACCUMULO_GC + ACCUMULO_TRACER + ACCUMULO_CLIENT + + + + + + + + + COMPLETE + + + + actionexecute + + + + + + UPGRADE + + + + + + + + true + false + + + UPGRADE + + Please confirm you are ready to finalize. + + + + + UPGRADE + + You can now remove any HBase snapshots which were created at the beginning of the upgrade. To see existing snapshots, use the following HBase shell command: + hbase> list_snapshots + Once you have found an existing snapshot which you would like to remove, you can use the following command: + hbase> delete_snapshot 'snapshotName' + + + + + + + finalize_rolling_upgrade + + + + + + + + + + + + diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml deleted file mode 100644 index 6ac5b68cbaf..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - 2.4.* - HDP-2.4.0 - ROLLING - - org.apache.ambari.server.checks.FooCheck - - - - HDFS - - - Back FOO up. - - - - - - ZOOKEEPER - - HBASE - - - - - HIVE - FOO - true - false - - BAR2 - - - - - HIVE - true - false - - BAR - - - - - YARN - - HBASE_MASTER - - - - - HDFS - - REGIONSERVER - - - 1 - Summary - Message - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml deleted file mode 100644 index 307f4d490be..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/config-upgrade.xml +++ /dev/null @@ -1,206 +0,0 @@ - - - - - - - - - - zookeeper-newconfig - - - - - zookeeper-newconfig - - - - - - - - - - - hdfs-site - - - - - hdfs-site - - - - - hdfs-site - - - - hdfs-newconfig - - - - - - - - - - - core-site - - - - - important-key - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hive-site - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml deleted file mode 100644 index 48433fef20f..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml +++ /dev/null @@ -1,98 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.6 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - ZOOKEEPER_SERVER - - - - - - - - - - - pre-manual 1 task to run on all hosts - - - - bar1 - - - - bar1 - - - - - - - - - - - bar1 - - - post-manual 1 task to run on all hosts - - - post-manual 2 task to run on all hosts - - - - bar2 - - - - bar3 - - - - bar4 - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_component_tasks_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_component_tasks_test.xml deleted file mode 100644 index 6cd9ce89f8c..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_component_tasks_test.xml +++ /dev/null @@ -1,133 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - NAMENODE - DATANODE - HDFS_CLIENT - JOURNALNODE - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - - - - - - - list - - - - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml deleted file mode 100644 index ab88df2ed54..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_direction.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.5 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - UPGRADE - - HDFS - YARN - - - - - - - - actionexecute - - - - - - - DOWNGRADE - - this is downgrade message - - - - - - - actionexecute - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml deleted file mode 100644 index 3e6169671bb..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - - - - - - bar - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_grouping_rolling.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_grouping_rolling.xml deleted file mode 100644 index 292976a2cff..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_grouping_rolling.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - - false - true - - ZOOKEEPER_SERVER - - - - - true - - ZOOKEEPER_SERVER - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_multi_server_tasks.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_multi_server_tasks.xml deleted file mode 100644 index de99d59bbe8..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_multi_server_tasks.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - - - true - false - - ZOOKEEPER_SERVER - - - - 20 - Please run additional tests on {{components}} - - - - - - true - false - - ZOOKEEPER_CLIENT - - - - - - - - - - - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - list - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml deleted file mode 100644 index 2a1c8897fb6..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml +++ /dev/null @@ -1,987 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0.1 - NON_ROLLING - - - - - - HDP-2.3.0.0 - - - - - - - UPGRADE - - true - false - - - - Before continuing, please stop all YARN queues. If yarn-site's yarn.resourcemanager.work-preserving-recovery.enabled is set to true, then you can skip this step since the clients will retry on their own. - - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - - - - prepare - - - - - - UPGRADE - false - true - false - - - - FLUME_HANDLER - - - - ACCUMULO_TRACER - ACCUMULO_GC - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_MASTER - - - - DRPC_SERVER - STORM_UI_SERVER - SUPERVISOR - NIMBUS - - - - KNOX_GATEWAY - - - - KAFKA_BROKER - - - - FALCON_SERVER - - - - OOZIE_SERVER - - - - SPARK_JOBHISTORYSERVER - - - - WEBHCAT_SERVER - HIVE_SERVER - HIVE_METASTORE - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - false - - - - Before continuing, please backup the Oozie Server database referenced by the Oozie server located on {{hosts.all}}. - - - - - - Before continuing, please backup the Hive Metastore database referenced by the Hive Metastore service(s) located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please backup the Knox data. E.g., "cp -RL /var/lib/knox/data/* ~/knox_backup/" on the following host(s): {{hosts.all}}. - - - - - - - take_snapshot - - - - - - - prepare_express_upgrade - - - - - - Before continuing, please backup the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - UPGRADE - false - true - false - - - - HBASE_REGIONSERVER - HBASE_MASTER - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - - - - RANGER_USERSYNC - RANGER_ADMIN - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - - Before continuing, please restore the Oozie Server database on {{hosts.all}}. - - - - - - Before continuing, please restore the Hive Metastore database located on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Knox data. E.g., "cp -RL ~/knox_backup/* /var/lib/knox/data/" on the following host(s): {{hosts.all}}. - - - - - - Before continuing, please restore the Ranger Admin database and Ranger Audit database on the following host(s): {{hosts.all}}. - - - - - - DOWNGRADE - - - - foo_function - - - - - - - - - - - - - - UPGRADE - true - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - true - false - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - false - - - RANGER_ADMIN - RANGER_USERSYNC - - - - - false - true - false - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - HDFS_CLIENT - - - - - false - true - - - DATANODE - - - - - UPGRADE - false - true - false - - - - Wait for NameNode to leave Safemode - - wait_for_safemode_off - - - - - - false - true - false - - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - YARN_CLIENT - - - - - false - true - - - - NODEMANAGER - - - - - false - true - false - - - HBASE_MASTER - HBASE_REGIONSERVER - HBASE_CLIENT - - - - - false - true - - - TEZ_CLIENT - - - - PIG - - - - SQOOP - - - - - UPGRADE - true - - - HDFS - YARN - MAPREDUCE2 - HBASE - - - - - false - true - false - - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - HIVE_CLIENT - HCAT - - - - - false - true - false - - - SPARK_JOBHISTORYSERVER - - - - - false - true - - - SPARK_CLIENT - - - - - - UPGRADE - true - false - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - - DOWNGRADE - true - - - Upgrading the Oozie database and creating a new sharelib - - create_sharelib - - - - - - false - true - false - - - OOZIE_SERVER - - - - - false - true - - - OOZIE_CLIENT - - - - - false - true - false - - - FALCON_SERVER - - - - - false - true - - - FALCON_CLIENT - - - - - false - true - - - KAFKA_BROKER - - - - - false - true - - - KNOX_GATEWAY - - - - - false - true - - - NIMBUS - SUPERVISOR - STORM_UI_SERVER - DRPC_SERVER - - - - - false - true - - - SLIDER - - - - - false - true - - - FLUME_HANDLER - - - - - false - true - - - ACCUMULO_MASTER - ACCUMULO_TSERVER - ACCUMULO_MONITOR - ACCUMULO_GC - ACCUMULO_TRACER - ACCUMULO_CLIENT - - - - - - true - false - - - - - actionexecute - - - - - - UPGRADE - - - - - - - - true - false - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - Create a new sharelib - - create_sharelib - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing Storm data from ZooKeeper - - delete_storm_zookeeper_data - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - Before continuing, please deactivate and kill any currently running topologies. - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - - - Removing local Storm data - - delete_storm_local_data - - - - - - - - - - - - Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar. - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml deleted file mode 100644 index b9733b60e02..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_server_action_test.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - - Foo - - - - - Back stuff up. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml deleted file mode 100644 index 037e39ac8bc..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - OOZIE_SERVER - OOZIE_CLIENT - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml deleted file mode 100644 index 6b08f272451..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_HDP-250.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - - 2.2.*.* - HDP-2.5.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - OOZIE_SERVER - OOZIE_CLIENT - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml deleted file mode 100644 index 6b91ee388c0..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_checks.xml +++ /dev/null @@ -1,232 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - global-value-1 - - ^([^:]*dfs|wasb|ecs):.* - - - ^([^:]*dfs|wasb|ecs):.* - - - - - - - - - Foo - - - - - - list - - - - - Foo - - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - false - - - - - HDFS - HBASE - YARN - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests - - - - - - ZOOKEEPER - HDFS - YARN - - - - - - OOZIE_SERVER - - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - list - - - - Update your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_force_config_change.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_force_config_change.xml deleted file mode 100644 index a438afc858c..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_force_config_change.xml +++ /dev/null @@ -1,267 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - OOZIE_SERVER - OOZIE_CLIENT - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_host_ordered.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_host_ordered.xml deleted file mode 100644 index 6238c829c78..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_host_ordered.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - - 2.2.* - HDP-2.2 - HOST_ORDERED - - - - - - - - Please confirm you are ready to accept this task - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml deleted file mode 100644 index 3dd4bc55061..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_nonrolling.xml +++ /dev/null @@ -1,200 +0,0 @@ - - - - - 2.2.*.* - HDP-2.2.3 - NON_ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - UPGRADE - true - - - - Before continuing, please stop all YARN queues. - - - - - - UPGRADE - false - true - - - - - NODEMANAGER - RESOURCEMANAGER - APP_TIMELINE_SERVER - - - - HISTORYSERVER - - - - - UPGRADE - true - - - - - prepare_non_rolling_upgrade - - - - - - UPGRADE - false - true - - - - - DATANODE - NAMENODE - SECONDARY_NAMENODE - ZKFC - JOURNALNODE - - - - ZOOKEEPER_SERVER - - - - - DOWNGRADE - true - - - - - restore_snapshot - - - - - - - - - - - - - - true - - - - actionexecute - - - - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - false - true - - 2 - - - JOURNALNODE - ZKFC - NAMENODE - SECONDARY_NAMENODE - DATANODE - HDFS_CLIENT - - - - - false - true - - - HISTORYSERVER - MAPREDUCE2_CLIENT - - - APP_TIMELINE_SERVER - RESOURCEMANAGER - NODEMANAGER - YARN_CLIENT - - - - - true - - - UPGRADE - - Please confirm you are ready to finalize. - - - - - - - finalize_non_rolling_upgrade - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_partial.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_partial.xml deleted file mode 100644 index 6e8745d25b3..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test_partial.xml +++ /dev/null @@ -1,278 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.0 - ROLLING - - - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - true - false - false - - OOZIE_SERVER - OOZIE_CLIENT - - - - - COMPLETE - - - Something to run on all hosts - - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - PARTIAL - Only partials - This is only for partials - - - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - - - - - Shut down all Oozie servers - - stop - - - - Upgrading the Oozie database and creating a new sharelib - - upgrade_oozie_database_and_sharelib - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml deleted file mode 100644 index fe387f39137..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_to_new_stack.xml +++ /dev/null @@ -1,229 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.4 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Foo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - - list - - - - - - - - - - - - list - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml deleted file mode 100644 index 385e539eafa..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/services/GANGLIA/upgrades/HDP/rolling-upgrade.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - * - * - ROLLING - - - ZOOKEEPER_CONFIG_CONDITION_TEST - true - true - - HST_SERVER - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml deleted file mode 100644 index dc7465e52fb..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/config-upgrade.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - zookeeper-newconfig - - - - - - - - - - - hdfs-site - - - - - - - - - - - core-site - - - - - important-key - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml deleted file mode 100644 index c819d1231e8..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test.xml +++ /dev/null @@ -1,220 +0,0 @@ - - - - 2.2.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Goo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - actionexecute - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml deleted file mode 100644 index 640ea0a2f65..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_15388.xml +++ /dev/null @@ -1,232 +0,0 @@ - - - - 2.4.* - HDP-2.4.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Goo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - actionexecute - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml deleted file mode 100644 index ef8ce704b60..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_checks.xml +++ /dev/null @@ -1,242 +0,0 @@ - - - - 2.2.*.* - HDP-2.2.1 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - global-value-1 - - ^([^:]*dfs|wasb|ecs):.* - - - ^([^:]*dfs|wasb|ecs):.* - - - - - - - - - Foo - - - - - - list - - - - - Foo - - - - - - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - false - - - - true - - - Update HBase Env Configuration - - - - - - - HDFS - YARN - - - AMBARI_METRICS - LOGSEARCH - - - - - true - false - - DATANODE - - - REGIONSERVER - - - NODEMANAGER - - - - 20 - Please run additional tests - - - - - - true - - - Update HBase Env Configuration - - - - - - - HDFS - YARN - - - AMBARI_METRICSS - - - - - - - - Please confirm you are ready to finalize - - - - - - list - - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - list - - - - Update your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - list - - - - - - - - - - list - - - - - - - - \ No newline at end of file diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_conditions.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_conditions.xml deleted file mode 100644 index b4b9c630628..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_conditions.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - 2.5.*.* - HDP-2.5.0 - ROLLING - - - - - - - Foo - - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - - - Foo - - - - - - - - - Foo - - - - - - - - Task based on security condition - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_host_ordered.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_host_ordered.xml deleted file mode 100644 index 6b9e086feee..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_host_ordered.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - - 2.2.* - HDP-2.2 - true - true - HOST_ORDERED - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - true - false - - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml b/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml deleted file mode 100644 index ed0fdda9c1a..00000000000 --- a/ambari-server/src/test/resources/stacks/HDP/2.2.0/upgrades/upgrade_test_skip_failures.xml +++ /dev/null @@ -1,78 +0,0 @@ - - - - 2.2.* - HDP-2.2 - true - true - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - true - false - - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml deleted file mode 100644 index 075cb13288d..00000000000 --- a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/services/HDFS/upgrades/HDP/2.2.0/upgrade_test_15388.xml +++ /dev/null @@ -1,97 +0,0 @@ - - - - 2.2.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.FooCheck - - - - HDFS - - - Back FOO up. - - - - - - ZOOKEEPER - - HBASE - - - - - BAR - true - false - - BAR - - - - - FOO - true - false - - FOO - - - - - YARN - - HBASE_MASTER - - - - - HDFS - - REGIONSERVER - - - 20 - Message - - - - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml deleted file mode 100644 index d40fabad24c..00000000000 --- a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/config-upgrade.xml +++ /dev/null @@ -1,88 +0,0 @@ - - - - - - - - - - zookeeper-newconfig - - - - - - - - - - - hdfs-site - - - - - - - - - - - core-site - - - - - important-key - - - - - - - - - - - hive-site - - - - - - - - - - - - - - - - - - - - diff --git a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/upgrade_test_15388.xml b/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/upgrade_test_15388.xml deleted file mode 100644 index 8a02781483c..00000000000 --- a/ambari-server/src/test/resources/stacks_with_upgrade_cycle/HDP/2.2.0/upgrades/upgrade_test_15388.xml +++ /dev/null @@ -1,239 +0,0 @@ - - - - 2.2.* - HDP-2.2.0 - ROLLING - - org.apache.ambari.server.checks.HiveMultipleMetastoreCheck - org.apache.ambari.server.checks.MapReduce2JobHistoryStatePreservingCheck - org.apache.ambari.server.checks.SecondaryNamenodeDeletedCheck - org.apache.ambari.server.checks.ServicesMapReduceDistributedCacheCheck - org.apache.ambari.server.checks.ServicesNamenodeHighAvailabilityCheck - org.apache.ambari.server.checks.ServicesNamenodeTruncateCheck - org.apache.ambari.server.checks.ServicesTezDistributedCacheCheck - org.apache.ambari.server.checks.ServicesYarnWorkPreservingCheck - org.apache.ambari.server.checks.YarnRMHighAvailabilityCheck - org.apache.ambari.server.checks.YarnTimelineServerStatePreservingCheck - - - - - - Foo - - - - - Back stuff up. - - - - - - list - - - - - Goo - - - - - - true - false - - ZOOKEEPER_SERVER - ZOOKEEPER_CLIENT - - - - - - JOURNALNODE - NAMENODE - - - RESOURCEMANAGER - - - - - true - false - - DATANODE - - - NODEMANAGER - - - - 20 - Please run additional tests on {{components}} - - - - - UPGRADE - true - - ZOOKEEPER - HDFS - YARN - - - AMBARI_METRICS - LOGSEARCH - - - - - true - - HIVE_METASTORE - HIVE_SERVER - WEBHCAT_SERVER - - - - - - - Please confirm you are ready to finalize - - - - - - actionexecute - - - - - - - - - - - - - - - - - SUMMARY OF PREPARE - This is a manual task with a placeholder of {{foo/bar}} - - - - - - - - - - - - - - - - - - - list - - - - - {{direction.verb.proper}} your database - - - - - - - - - - list - - - - - - - - - - - - Manual Downgrade - - - - - - - - - - - list - - - - - - - - - - - - list - - - - - - - - - - - - - - HiveServer Port Availability - The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated. - - - - - - - - - - - diff --git a/mpack-instance-manager/pom.xml b/mpack-instance-manager/pom.xml new file mode 100644 index 00000000000..8df08fa056b --- /dev/null +++ b/mpack-instance-manager/pom.xml @@ -0,0 +1,253 @@ + + + + + org.apache.ambari + ambari-project + 2.0.0.0-SNAPSHOT + ../ambari-project + + 4.0.0 + org.apache.ambari + mpack-instance-manager + 2.0.0.0-SNAPSHOT + Mpack instance manager + Mpack Instance Manager + + /usr/lib/mpack-instance-manager + UTF-8 + ${project.artifactId}-${project.version} + 1 + /usr + false + amd64 + ${project.build.directory}/cache/ + [Tt]est*.py + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.8 + + + parse-version + validate + + parse-version + + + + regex-property + + regex-property + + + ambariVersion + ${project.version} + ^([0-9]+)\.([0-9]+)\.([0-9]+)\.([0-9]+)(\.|-).* + $1.$2.$3.$4 + false + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + + ${executable.python} + src/test/python + + unitTests.py + + + ../../main/python:$PYTHONPATH + + ${skipTests} + + python-test + test + + exec + + + + + + org.codehaus.mojo + rpm-maven-plugin + 2.1.4 + + + + none + + rpm + + + + + + 2012, Apache Software Foundation + Development + Maven Recipe: RPM Package. + + src/main/package/rpm/postinstall.sh + utf-8 + + + src/main/package/rpm/preremove.sh + utf-8 + + + x86_64 + false + + + ${instanceManagerInstallDir} + 755 + root + root + + + + ${project.build.directory}${dirsep}${project.artifactId}-${project.version}${instanceManagerInstallDir} + + + + + + + + + org.vafer + jdeb + 1.4 + + + none + + jdeb + + + + + ${basedir}/src/main/package/deb/control + ${basedir}/target/${project.artifactId}_${package-version}-${package-release}.deb + false + false + + + ${project.build.directory}${dirsep}${project.artifactId}-${project.version}.tar.gz + + archive + + perm + root + root + + + + + + + maven-assembly-plugin + + gnu + + src/packages/tarball/all.xml + + + + + make-assembly + ${assemblyPhase} + + single + + + + + + org.apache.rat + apache-rat-plugin + + + test + + check + + + + + + org.codehaus.mojo + buildnumber-maven-plugin + ${buildnumber-maven-plugin-version} + + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-ambari.git + + + + validate + + create + + + + + + + + org.apache.maven.wagon + wagon-ssh-external + + + + + + linux + + + unix + + + + linux + / + : + ${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap + + sh + sh + + + ${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/ambari_jinja2:${project.basedir}/../ambari-common/src/main/python/ambari_commons:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari_agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python:${project.basedir}/src/test/python/ambari_agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/../ambari-server/src/test/python:${project.basedir}/../ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files + + + + + \ No newline at end of file diff --git a/mpack-instance-manager/src/main/package/deb/control/control b/mpack-instance-manager/src/main/package/deb/control/control new file mode 100644 index 00000000000..2ea34a51d6c --- /dev/null +++ b/mpack-instance-manager/src/main/package/deb/control/control @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +Package: [[artifactId]] +Version: [[package-version]]-[[package-release]] +Section: [[deb.section]] +Priority: [[deb.priority]] +Depends: [[deb.dependency.list]] +Architecture: [[deb.architecture]] +Description: [[description]] +Maintainer: [[deb.publisher]] \ No newline at end of file diff --git a/mpack-instance-manager/src/main/package/deb/control/postinst b/mpack-instance-manager/src/main/package/deb/control/postinst new file mode 100644 index 00000000000..e520da49d48 --- /dev/null +++ b/mpack-instance-manager/src/main/package/deb/control/postinst @@ -0,0 +1,25 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +MPACK_INSTANCE_MANAGER_BINARY="/usr/lib/mpack-instance-manager/mpack-instance-manager.py" +MPACK_INSTANCE_MANAGER_BINARY_SYMLINK="/usr/sbin/mpack-instance-manager" + +# setting mpack-instance-manager binary symlink +if [ ! -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" ]; then + ln -s "$MPACK_INSTANCE_MANAGER_BINARY" "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" +fi + +exit 0 \ No newline at end of file diff --git a/mpack-instance-manager/src/main/package/deb/control/prerm b/mpack-instance-manager/src/main/package/deb/control/prerm new file mode 100644 index 00000000000..5b36d4e40d8 --- /dev/null +++ b/mpack-instance-manager/src/main/package/deb/control/prerm @@ -0,0 +1,29 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# WARNING: This script is performed not only on uninstall, but also +# during package update. See http://www.ibm.com/developerworks/library/l-rpm2/ +# for details + +MPACK_INSTANCE_MANAGER_BINARY_SYMLINK="/usr/sbin/mpack-instance-manager" + +if [ "$1" -eq 0 ]; then # Action is uninstall + if [ -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" ]; then + rm -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" + fi +fi + +exit 0 \ No newline at end of file diff --git a/mpack-instance-manager/src/main/package/rpm/postinstall.sh b/mpack-instance-manager/src/main/package/rpm/postinstall.sh new file mode 100644 index 00000000000..d94ca6038f6 --- /dev/null +++ b/mpack-instance-manager/src/main/package/rpm/postinstall.sh @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +MPACK_INSTANCE_MANAGER_BINARY="/usr/lib/mpack-instance-manager/mpack-instance-manager.py" +MPACK_INSTANCE_MANAGER_BINARY_SYMLINK="/usr/sbin/mpack-instance-manager" + +# setting mpack-instance-manager binary symlink +if [ ! -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" ]; then + ln -s "$MPACK_INSTANCE_MANAGER_BINARY" "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" +fi + +exit 0 \ No newline at end of file diff --git a/mpack-instance-manager/src/main/package/rpm/preremove.sh b/mpack-instance-manager/src/main/package/rpm/preremove.sh new file mode 100644 index 00000000000..7101a02c3ef --- /dev/null +++ b/mpack-instance-manager/src/main/package/rpm/preremove.sh @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# WARNING: This script is performed not only on uninstall, but also +# during package update. See http://www.ibm.com/developerworks/library/l-rpm2/ +# for details + +MPACK_INSTANCE_MANAGER_BINARY_SYMLINK="/usr/sbin/mpack-instance-manager" + +if [ "$1" -eq 0 ]; then # Action is uninstall + if [ -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" ]; then + rm -f "$MPACK_INSTANCE_MANAGER_BINARY_SYMLINK" + fi +fi + +exit 0 \ No newline at end of file diff --git a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py new file mode 100644 index 00000000000..668223428f2 --- /dev/null +++ b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py @@ -0,0 +1,629 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +__all__ = ["create_mpack", "set_mpack_instance", "get_conf_dir", "list_instances"] + +import sys +import os +import json + +MPACK_JSON_FILE_NAME = 'mpack.json' +CURRENT_SOFTLINK_NAME = 'current' +CONFIGS_DIRECTORY_NAME = 'conf' + +ROOT_FOLDER_PATH = "/usr/hwx/" + +INSTANCES_FOLDER_NAME = "instances" +MODULES_FOLDER_NAME = "modules" +MPACKS_FOLDER_NAME = "mpacks" +DEFAULT_COMPONENT_INSTANCE_NAME = 'default' +DEFAULT_MPACK_INSTANCE_NAME = 'default' +DEFAULT_SUBGROUP_NAME = 'default' +CLIENT_CATEGORY = "CLIENT" +SERVER_CATEGORY = "SERVER" + + +def create_mpack(mpack_name, mpack_version, mpack_instance, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components=None, components_map=None): + """ + Use case 1: Creates an instance of mpack with a new subgroup, new module and one or more component(s) + + Use case 2: Creates/adds in an existing mpack either a : new subgroup and/or module and/or component instance. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map to create single instance of all components with name 'default' + OR + list of 'components_instances_name' to be created. (default) OR '*' for all components + """ + mpack_name = mpack_name.lower() + module_name = module_name.lower() + + validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map) + + MpackInstance.create_mpack_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, module_name, + components, components_map) + + +def set_mpack_instance(mpack, mpack_version, mpack_instance, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components=None, components_map=None): + """ + Use case: move a given component instances from one version to the next version by modifying the soft links. + Eg: Moving hive_server instance 'HS-1' from version 1.0.0-b1 to 1.5.0-b1. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map to create single instance of all components with name 'default' + OR + list of 'components_instances_name' to be created. (default) OR '*' for all components + """ + mpack = mpack.lower() + module_name = module_name.lower() + + instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME), mpack, + mpack_instance, subgroup_name, module_name, + components, components_map) + if not instances: + raise ValueError("Found no instances for the given filters.") + + validate_mpack_for_creation_or_changing(mpack, mpack_version, module_name, components, components_map) + + for mpack in instances: + for mpack_instance_name in instances[mpack]: + instances[mpack][mpack_instance_name].set_new_version(mpack, mpack_version) + + +def get_conf_dir(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components_map=None): + """ + Use case: retrieve conf directory paths for a given component instances based on the granularity specified + ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance + AND with a filtering on each level + + Granularity works only while names for all consecutive levels are specified. + Note that subgroup has default value of 'default' + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output + of subgroup will be returned. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map for all component instances present + """ + return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map, + output_conf_dir=True) + + +def list_instances(mpack=None, mpack_instance=None, subgroup_name=DEFAULT_SUBGROUP_NAME, module_name=None, + components_map=None): + """ + Use case: figure out the versions a given component instances based on the granularity specified + ranging from: mpack, mpack-instance, subgroup-name, module-name and map of component instance + AND with a filtering on each level + + Granularity works only while names for all consecutive levels are specified. + Note that subgroup has default value of 'default' + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output + of subgroup will be returned. + + Components are provided as map with key as 'component type' and value as 'list of individual component instances + names' OR empty map for all component instances present + """ + return build_granular_json_with_filtering(mpack, mpack_instance, subgroup_name, module_name, components_map, + output_path=True) + + +def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter, subgroup_name_filter, + module_name_filter, components_name_filter_map, output_conf_dir=False, + output_path=False): + """ + Builds the json that contains all instances specified in filters or all instances if filters are not specified. + The level of granularity depends on the consecutive levels of specified filters + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + though the subgroup fitler will be applied. But if the instance name is specified also, than only granular output + of subgroup will be returned. + + The output_conf_dir or output_path for each component instance will be included in json depending on given parameters. + """ + + if mpack_name_filter: + mpack_name_filter = mpack_name_filter.lower() + if module_name_filter: + module_name_filter = module_name_filter.lower() + + instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME), + mpack_name_filter, + instance_name_filter, subgroup_name_filter, + module_name_filter, + None, + components_name_filter_map) + if not instances: + raise ValueError("Found no instances for the given filters.") + + full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path) + + granular_json_output = build_granular_output(full_json_output, mpack_name_filter, instance_name_filter, + subgroup_name_filter, + module_name_filter) + + return granular_json_output + + +def build_granular_output(json_output, mpack_name_filter, instance_name_filter, subgroup_name_filter, + module_name_filter): + """ + Returns the part of original json using the granularity filters + + The level of granularity depends on the consecutive levels of specified filters + Levels: mpack/instance/subgroup/module + E.g If only mpack and subgroup names are specified, the granularity will work only on mpack level, + But if the instance name is specified also, than only granular output of subgroup will be returned. + """ + if mpack_name_filter: + json_output = json_output['mpacks'][mpack_name_filter] + if instance_name_filter: + json_output = json_output[MpackInstance.plural_name][instance_name_filter] + if subgroup_name_filter: + json_output = json_output['subgroups'][subgroup_name_filter] + if module_name_filter: + json_output = json_output[ModuleInstance.plural_name][module_name_filter] + return json_output + + +def build_json_output_from_instances_dict(instances_dict, plural_name, output_conf_dir, output_path): + """ + Build the json from the dictionary of Instance objects. + The plural_name is used to form the upper level of the json output. + """ + result = {} + for instance_name in instances_dict: + result[instance_name] = instances_dict[instance_name].build_json_output(output_conf_dir, output_path) + + return {plural_name: result} + + +# Better use it with component instance path. +# Need to be careful with this method as it will return only the single meta mpack whilst the module component instances +# may be from different mpack versions and point to different mpack.json's +def find_link_to_current_in_path_recursive(root_path): + if not os.path.lexists(os.path.join(root_path, CURRENT_SOFTLINK_NAME)): + walk = os.walk(root_path) + for folder_name in next(walk)[1]: + folder_search_result = find_link_to_current_in_path_recursive(os.path.join(root_path, folder_name)) + if folder_search_result: + return folder_search_result + else: + return None + else: + return os.path.join(root_path, CURRENT_SOFTLINK_NAME) + + +def get_module_meta_mpack(path, module_name): + current_link_location = find_link_to_current_in_path_recursive(os.path.join(path, module_name)) + current_target = os.readlink(current_link_location) + return MetaMpack.parse_mpack(os.path.dirname(current_target)) + + +def build_json_output(instances, output_conf_dir=False, output_path=False): + result = {} + for mpack_name in instances.keys(): + result[mpack_name] = build_json_output_from_instances_dict(instances[mpack_name], MpackInstance.plural_name, + output_conf_dir, output_path) + return {'mpacks': result} + + +def validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components, components_map): + mpack_root_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name) + if not os.path.exists(mpack_root_path): + raise ValueError("Mpack {0} doesn't exist, please check mpack name.".format(mpack_name)) + + mpack_version_path = os.path.join(mpack_root_path, mpack_version) + if not os.path.exists(mpack_version_path): + raise ValueError( + "Mpack version {0} doesn't exist for mpack {1}, please check mpack name and version".format(mpack_version, + mpack_name)) + + meta_mpack = MetaMpack.parse_mpack(mpack_version_path) + + if not module_name in meta_mpack.module_component_types_map: + raise ValueError( + "There is no module {0} for mpack {1} with version {2}. Please check mpack name, version and module name".format( + module_name, mpack_name, mpack_version)) + + if components and components != "*": + for component in components: + if not meta_mpack.get_component_category(component): + raise ValueError( + "There is no component {0} in module {1} for mpack {2} with version {3}." + " Please check mpack name, version, module name and component name".format( + component, module_name, mpack_name, mpack_version)) + + if components_map: + for component in components_map: + if component not in meta_mpack.module_component_types_map[module_name]: + raise ValueError( + "There is no component {0} in module {1} for mpack {2} with version {3}." + " Please check mpack name, version, module name and component name".format( + component, module_name, mpack_name, mpack_version)) + + +class MetaMpack: + def __init__(self, name, version, component_module_map, mpack_json): + self.name = name + self.version = version + self.component_module_map = component_module_map + self.mpack_json = mpack_json + self.module_categoty_map = self.parse_mpack_json_into_module_category_map() + self.module_component_types_map = self.parse_mpack_json_into_module_component_types_map() + + def parse_mpack_json_into_module_category_map(self): + result = {} + if not self.mpack_json: + return None + for module in self.mpack_json['modules']: + result[module['id']] = module['category'] + + return result + + def parse_mpack_json_into_module_component_types_map(self): + result = {} + if not self.mpack_json: + return None + for module in self.mpack_json['modules']: + result[module['id']] = [] + for component in module['components']: + result[module['id']].append(component['id']) + + return result + + def get_component_category(self, component_type): + for module in self.module_component_types_map: + if component_type in self.module_component_types_map[module]: + return self.module_categoty_map[module] + return None + + @staticmethod + def parse_mpack(path): + # build components map from soft links + component_module_map = {} + for filename in os.listdir(path): + if os.path.islink(os.path.join(path, filename)): + component_module_map[filename] = os.path.realpath(os.path.join(path, filename)) + + mpack_json_path = os.path.join(path, MPACK_JSON_FILE_NAME) + if not os.path.exists(mpack_json_path): + raise ValueError( + "{0} file is missing. The exact location should be {1}".format(MPACK_JSON_FILE_NAME, mpack_json_path)) + + with open(mpack_json_path, "r") as json_file: + json_file_content = json_file.read() + + try: + mpack_json = json.loads(json_file_content) + except ValueError as e: + raise ValueError("The {0} is invalid. Location: {1}. Error message: {2}".format( + MPACK_JSON_FILE_NAME, mpack_json_path, e.message)) + + mpack_version = os.path.basename(path) + mpack_name = os.path.basename(os.path.dirname(path)) + return MetaMpack(name=mpack_name, + version=mpack_version, + component_module_map=component_module_map, + mpack_json=mpack_json) + + @staticmethod + def parse_mpacks(path): + result = {} + walk = os.walk(path) + for mpack_name in next(walk)[1]: + result[mpack_name] = MetaMpack.parse_into_mpack_objects(os.path.join(path, mpack_name)) + return result + + @staticmethod + def parse_into_mpack_objects(path): + result = {} + walk = os.walk(path) + for mpack_version in next(walk)[1]: + result[mpack_version] = MetaMpack.parse_mpack(path=os.path.join(path, mpack_version)) + return result + + +class Instance: + def build_json_output(self, output_conf_dir, output_path): + raise NotImplementedError("Should have implemented this") + + +class MpackInstance(Instance): + plural_name = "mpack-instances" + + def __init__(self, mpack_name, instance_name, groups_dict): + self.mpack_name = mpack_name + self.instance_name = instance_name + self.groups_dict = groups_dict + + def build_json_output(self, output_conf_dir, output_path): + result = {} + for group in self.groups_dict.keys(): + result[group] = build_json_output_from_instances_dict(self.groups_dict[group], ModuleInstance.plural_name, + output_conf_dir, output_path) + return {"subgroups": result, 'name': self.instance_name} + + def set_new_version(self, mpack_name, mpack_version): + for subgroup_name in self.groups_dict: + for module_name in self.groups_dict[subgroup_name]: + self.groups_dict[subgroup_name][module_name].set_new_version(mpack_name, mpack_version) + + @staticmethod + def parse_into_mpack_instance_dict(path, mpack_name, instance_name_filter, subgroup_name_filter, + module_name_filter, components_filter, components_name_filter_map): + result = {} + walk = os.walk(os.path.join(path, mpack_name)) + for instance_name in next(walk)[1]: + if ( + not instance_name_filter or instance_name_filter == instance_name) and instance_name != DEFAULT_MPACK_INSTANCE_NAME: + mpack_instance_object = MpackInstance.parse_into_mpack_instance_object(path, mpack_name, + instance_name, + subgroup_name_filter, + module_name_filter, + components_filter, + components_name_filter_map) + if mpack_instance_object: + result[instance_name] = mpack_instance_object + return result + + @staticmethod + def create_mpack_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, module_name, + components, + components_map): + ModuleInstance.create_module_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, + module_name, components, components_map) + + default_mpack_instance_symlink = os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME, mpack_name, + DEFAULT_MPACK_INSTANCE_NAME) + if not os.path.lexists(default_mpack_instance_symlink): + os.symlink(os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME, mpack_name, mpack_instance), + default_mpack_instance_symlink) + + @staticmethod + def parse_instances_with_filtering(path, mpack_name_filter=None, instance_name_filter=None, subgroup_name_filter=None, + module_name_filter=None, components_filter=None, components_name_filter_map=None): + if not os.path.exists(path): + raise ValueError("There are no created instances. Use create-mpack-instance command to add them.") + result = {} + walk = os.walk(path) + for mpack_name in next(walk)[1]: + if not mpack_name_filter or mpack_name_filter == mpack_name: + mpack_instance_dict = MpackInstance.parse_into_mpack_instance_dict(path, mpack_name, + instance_name_filter, + subgroup_name_filter, + module_name_filter, + components_filter, + components_name_filter_map) + if mpack_instance_dict: + result[mpack_name] = mpack_instance_dict + return result + + @staticmethod + def parse_into_mpack_instance_object(root_path, mpack_name, instance_name, subgroup_name_filter=None, + module_name_filter=None, components_filter=None, + components_name_filter_map=None): + full_path = os.path.join(root_path, mpack_name, instance_name) + + # return None if instance doesn't exist + if not os.path.exists(full_path): + return None + + # build groups dictionary + groups_dict = {} + walk = os.walk(full_path) + for group_name in next(walk)[1]: + if not subgroup_name_filter or subgroup_name_filter == group_name: + module_instance_dict = ModuleInstance.parse_into_module_instance_dict( + os.path.join(full_path, group_name), module_name_filter, components_filter, components_name_filter_map) + if module_instance_dict: + groups_dict[group_name] = module_instance_dict + + if not groups_dict: + return None + return MpackInstance(mpack_name, instance_name, groups_dict) + + +class ModuleInstance(Instance): + plural_name = "modules" + + def __init__(self, module_name, components_map, category): + self.module_name = module_name + self.components_map = components_map + self.category = category + + @staticmethod + def parse_into_module_instance_dict(path, module_name_filter, components_filter, components_name_filter_map): + result = {} + walk = os.walk(path) + # The problem here is that for client component this folder name means the component type, while for server modules + # it's module name + for folder_name in next(walk)[1]: + components_map = None + + # find out the module category and name + meta_mpack = get_module_meta_mpack(path, folder_name) + if folder_name in meta_mpack.module_categoty_map: + module_category = meta_mpack.module_categoty_map[folder_name] + module_name = folder_name + else: + module_category = meta_mpack.get_component_category(folder_name) + module_name = os.path.basename(os.path.dirname(meta_mpack.component_module_map[folder_name])) + + if not module_name_filter or module_name == module_name_filter: + if module_category == CLIENT_CATEGORY: + if ((not components_filter and not components_name_filter_map) or + (components_filter and (components_filter == '*' or folder_name in components_filter)) or + (components_name_filter_map and folder_name in components_name_filter_map)): + components_map = ComponentInstance(name=DEFAULT_COMPONENT_INSTANCE_NAME, + component_path=os.path.join(path, folder_name), + path_exec=os.path.realpath( + os.path.join(path, folder_name, CURRENT_SOFTLINK_NAME))) + else: + components_map = ComponentInstance.parse_into_components_dict(os.path.join(path, folder_name), + components_filter, + components_name_filter_map) + if components_map: + result[folder_name] = ModuleInstance(folder_name, components_map, module_category) + return result + + @staticmethod + def create_module_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, module_name, + components, + components_map): + meta_mpack = MetaMpack.parse_mpack( + path=os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name, mpack_version)) + + is_client_module = meta_mpack.module_categoty_map[module_name] == CLIENT_CATEGORY + if components: + if components == '*': + components = meta_mpack.module_component_types_map[module_name] + for component_type in components: + ComponentInstance.create_component_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, + module_name, component_type, DEFAULT_COMPONENT_INSTANCE_NAME, + is_client_module) + else: + for component_type in components_map: + for component_instance_name in components_map[component_type]: + ComponentInstance.create_component_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, + module_name, component_type, component_instance_name, + is_client_module) + + def set_new_version(self, mpack_name, mpack_version): + if self.category == CLIENT_CATEGORY: + component_instance = self.components_map + print("\nSetting new version for component : " + component_instance.component_path) + component_instance.set_new_version(mpack_name, mpack_version, self.module_name) + else: + for component_type in self.components_map: + for component_name in self.components_map[component_type]: + component_instance = self.components_map[component_type][component_name] + print("\nSetting new version for component : " + component_instance.component_path) + component_instance.set_new_version(mpack_name, mpack_version, component_type) + + def build_json_output(self, output_conf_dir, output_path): + result = {} + if self.category == CLIENT_CATEGORY: + result['component_instances'] = {'default': self.components_map.build_json_output(output_conf_dir, output_path)} + else: + for component_type in self.components_map.keys(): + result[component_type] = build_json_output_from_instances_dict(self.components_map[component_type], + ComponentInstance.plural_name, + output_conf_dir, output_path) + result = {'components': result} + + result['category'] = self.category + result['name'] = self.module_name + return result + + +class ComponentInstance(Instance): + plural_name = "component-instances" + + def __init__(self, name, component_path, path_exec): + self.name = name + self.component_path = component_path + self.path_exec = path_exec + + def set_new_version(self, mpack_name, mpack_version, component_type): + mpack_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name, mpack_version, component_type) + target_link = os.path.join(self.component_path, CURRENT_SOFTLINK_NAME) + if os.path.lexists(target_link): + if os.readlink(target_link) == mpack_path: + print "\n{0} already points to {1}. Skipping.".format(target_link, mpack_path) + return + os.remove(target_link) + + os.symlink(mpack_path, target_link) + print "\nCreated new link " + target_link + " -> " + mpack_path + + @staticmethod + def parse_into_component_instance_dict(path, component_names_filter=None): + result = {} + walk = os.walk(path) + for component_instance_name in next(walk)[1]: + if not component_names_filter or component_instance_name in component_names_filter: + result[component_instance_name] = ComponentInstance(name=component_instance_name, + component_path=os.path.join(path, component_instance_name), + path_exec=os.path.realpath( + os.path.join(path, component_instance_name, + CURRENT_SOFTLINK_NAME))) + return result + + @staticmethod + def parse_into_components_dict(path, components_filter, components_name_filter_map): + result = {} + walk = os.walk(path) + for component_type in next(walk)[1]: + if components_filter: + if components_filter == '*': + result[component_type] = ComponentInstance.parse_into_component_instance_dict( + os.path.join(path, component_type)) + elif component_type in components_filter: + component_instance_dict = ComponentInstance.parse_into_component_instance_dict( + os.path.join(path, component_type)) + if component_instance_dict: + result[component_type] = component_instance_dict + elif not components_name_filter_map: + result[component_type] = ComponentInstance.parse_into_component_instance_dict( + os.path.join(path, component_type)) + elif component_type in components_name_filter_map.keys(): + component_instance_dict = ComponentInstance.parse_into_component_instance_dict( + os.path.join(path, component_type), components_name_filter_map[component_type]) + if component_instance_dict: + result[component_type] = component_instance_dict + return result + + @staticmethod + def create_component_instance(mpack_name, mpack_version, mpack_instance, subgroup_name, module_name, + component_type, component_instance_name, is_client_module): + if is_client_module: + component_path = os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME, mpack_name, mpack_instance, subgroup_name, + component_type) + else: + component_path = os.path.join(ROOT_FOLDER_PATH, INSTANCES_FOLDER_NAME, mpack_name, mpack_instance, subgroup_name, + module_name, component_type, component_instance_name) + mpack_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name, mpack_version, component_type) + + if os.path.exists(component_path): + raise ValueError( + "The instance {0} already exist. To change the version use set-mpack-instance command".format(component_path)) + + if not os.path.lexists(mpack_path): + raise ValueError("Path doesn't exist: " + mpack_path) + + os.makedirs(component_path) + + os.symlink(mpack_path, os.path.join(component_path, CURRENT_SOFTLINK_NAME)) + + os.makedirs(os.path.join(component_path, CONFIGS_DIRECTORY_NAME)) + print "\n Created " + component_path + + def build_json_output(self, output_conf_dir, output_path): + result = {'name': self.name} + if output_conf_dir: + result['config_dir'] = os.path.join(self.component_path, CONFIGS_DIRECTORY_NAME) + if output_path: + result['path'] = self.path_exec + return result diff --git a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py new file mode 100644 index 00000000000..38c29f827f1 --- /dev/null +++ b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from instance_manager import * + +import optparse +import sys +import ast + +CREATE_MPACK_INSTANCE_ACTION = 'create-mpack-instance' +SET_MPACK_INSTANCE_ACTION = 'set-mpack-instance' +GET_CONF_DIR_ACTION = 'get-conf-dir' +LIST_INSTANCES_ACTION = 'list-instances' + + +def init_action_parser(action, parser): + action_parser_map = { + CREATE_MPACK_INSTANCE_ACTION: init_create_parser_options, + SET_MPACK_INSTANCE_ACTION: init_set_parser_options, + GET_CONF_DIR_ACTION: init_get_parser_options, + LIST_INSTANCES_ACTION: init_get_parser_options + } + try: + action_parser_map[action](parser) + except KeyError: + parser.error( + "Invalid action {0}. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances".format( + action)) + + +def check_required_options(options, action, parser): + missing_options = [] + if action == CREATE_MPACK_INSTANCE_ACTION or action == SET_MPACK_INSTANCE_ACTION: + if not options.mpack: + missing_options.append("mpack") + if not options.mpack_version: + missing_options.append("mpack-version") + if not options.mpack_instance: + missing_options.append("mpack-instance") + if not options.module_name: + missing_options.append("module-name") + + if missing_options: + parser.error("Missing following required command options: {0}".format(missing_options)) + + +def init_create_parser_options(parser): + parser.add_option('--mpack', default=None, help="selected 'mpack_name'. eg: edw, core", dest="mpack") + parser.add_option('--mpack-version', default=None, help="selected 'mpack_version'. eg. 1.0.0.-b1", + dest="mpack_version") + parser.add_option('--mpack-instance', default=None, + help="new/existing 'mpack_instance_name'. eg: Production, eCommerce", dest="mpack_instance") + parser.add_option('--subgroup-name', default="default", help="new/existing 'subgroup_name' eg: hive-for-finance", + dest="subgroup_name") + parser.add_option('--module-name', default=None, + help="selected 'service/client_module' eg: hive, hdfs, hive_client etc", dest="module_name") + parser.add_option('--components', default=None, + help="list of 'components_instances_name' to be created. (default) OR '*' for all components", + dest="components") + parser.add_option('--components-map', default=None, + help="map of 'component type' (eg: hive_server, metastore etc) as key and List of component instance name(s) to be given (eg: HS-1, finance_metastore) as value OR Empty map to create single instance of all components with name 'default'", + dest="components_map") + + +def init_set_parser_options(parser): + parser.add_option('--mpack', default=None, help="selected 'mpack_name'. eg: edw, core", dest="mpack") + parser.add_option('--mpack-version', default=None, + help="selected 'mpack_version' on which we want to move to. eg. 1.5.0.-b1", dest="mpack_version") + parser.add_option('--mpack-instance', default=None, help="existing 'mpack_instance_name'. eg: Production, eCommerce", + dest="mpack_instance") + parser.add_option('--subgroup-name', default="default", help="existing 'subgroup_name' eg: hive-for-finance", + dest="subgroup_name") + parser.add_option('--module-name', default=None, + help="selected 'service/client_module' eg: hive, hdfs, hive_client etc", dest="module_name") + parser.add_option('--components', default=None, + help="list of 'components_instances_name' to be updated. (default) OR '*' for all components", + dest="components") + parser.add_option('--components-map', default=None, + help="map of 'component type' (eg: hive_server, metastore etc) as key and List of component instance name(s) to be given (eg: HS-1, finance_metastore) as value OR Empty map to update instance of all components with name 'default'", + dest="components_map") + + +def init_get_parser_options(parser): + parser.add_option('--mpack', default=None, + help="'mpack_name' to which component instance belongs. eg: edw, core'. eg: edw, core", + dest="mpack") + parser.add_option('--mpack-instance', default=None, help="'mpack_instance_name'. (eg: default)", + dest="mpack_instance") + parser.add_option('--subgroup-name', default="default", help="'subgroup_name' eg: hive-for-finance", + dest="subgroup_name") + parser.add_option('--module-name', default=None, + help="selected 'service/client_module' eg: hive, hdfs, hive_client etc", dest="module_name") + parser.add_option('--components-map', default=None, + help="map of 'component type' (eg: hive_server, metastore etc) as key and List of component instance name(s) to be given (eg: HS-1, finance_metastore) as value OR Empty map for all component instances present", + dest="components_map") + + +def main(options, args): + action = sys.argv[1] + + parsed_components = None + parsed_components_map = None + try: + if hasattr(options, 'components') and options.components and options.components != '*': + parsed_components = ast.literal_eval(options.components) + if options.components_map: + parsed_components_map = ast.literal_eval(options.components_map) + except ValueError: + raise ValueError("Components or components-map format is invalid.") + + if action == CREATE_MPACK_INSTANCE_ACTION: + create_mpack(mpack_name=options.mpack, mpack_version=options.mpack_version, + mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components=parsed_components, + components_map=parsed_components_map) + + elif action == SET_MPACK_INSTANCE_ACTION: + set_mpack_instance(mpack=options.mpack, mpack_version=options.mpack_version, + mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components=parsed_components, + components_map=parsed_components_map) + + elif action == GET_CONF_DIR_ACTION: + print get_conf_dir(mpack=options.mpack, mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components_map=parsed_components_map) + + elif action == LIST_INSTANCES_ACTION: + print list_instances(mpack=options.mpack, mpack_instance=options.mpack_instance, + subgroup_name=options.subgroup_name, module_name=options.module_name, + components_map=parsed_components_map) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print( + "Missing the command. Possible options are: create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances") + sys.exit(1) + + usage = "Usage: %prog create-mpack-instance|set-mpack-instance|get-conf-dir|list-instances [options]" + parser = optparse.OptionParser(usage=usage) + action = sys.argv[1] + init_action_parser(action, parser) + (options, args) = parser.parse_args() + check_required_options(options, action, parser) + + try: + main(options, args) + except (KeyboardInterrupt, EOFError): + print("\nAborting ... Keyboard Interrupt.") + sys.exit(1) + except ValueError as e: + print "ERROR: " + e.message + sys.exit(1) diff --git a/mpack-instance-manager/src/packages/tarball/all.xml b/mpack-instance-manager/src/packages/tarball/all.xml new file mode 100644 index 00000000000..1c90f2824a5 --- /dev/null +++ b/mpack-instance-manager/src/packages/tarball/all.xml @@ -0,0 +1,37 @@ + + + + + + dir + tar.gz + + false + + + 755 + 755 + src/main/python/instance_manager + ${instanceManagerInstallDir} + + + diff --git a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py new file mode 100644 index 00000000000..53150941526 --- /dev/null +++ b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" +from unittest import TestCase + +import instance_manager +import os +import shutil +import json + +TMP_ROOT_FOLDER = "/tmp/instance_manager_test" +# set root directory to tmp location +instance_manager.ROOT_FOLDER_PATH = TMP_ROOT_FOLDER + +MPACK_NAME = 'hdpcore' +MPACK_NAME2 = 'edw' +MPACK_VERSION_1 = '1.0.0-b1' +MPACK_VERSION_2 = '1.5.0-b1' +INSTANCE_NAME_1 = 'Production' +INSTANCE_NAME_2 = 'eCommerce' +SUBGROUP_NAME = 'default' +CLIENT_MODULE_NAME = 'hdfs-clients' +CLIENT_COMPONENT_NAME = 'hdfs_client' +SERVER_MODULE_NAME = 'hdfs' +SERVER_COMPONENT_NAME = 'hdfs_server' +MODULE_VERSION_MAPPING = {CLIENT_MODULE_NAME: '3.1.0.0-b1', SERVER_MODULE_NAME: '3.1.0.0-b1'} +MODULE_COMPONENT_MAPPING = {CLIENT_MODULE_NAME: CLIENT_COMPONENT_NAME, SERVER_MODULE_NAME: SERVER_COMPONENT_NAME} + +MPACK_JSON = {"modules": [ + {"category": "CLIENT", "components": [{"id": CLIENT_COMPONENT_NAME}], "id": CLIENT_MODULE_NAME}, + {"category": "SERVER", "components": [{"id": SERVER_COMPONENT_NAME}], "id": SERVER_MODULE_NAME}]} + + +class TestInstanceManager(TestCase): + def setUp(self): + build_rpm_structure() + + def tearDown(self): + remove_rpm_structure() + + def test_create_mpack_client_module(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME) + current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, CLIENT_COMPONENT_NAME, + instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertTrue(os.path.exists(current_link)) + self.assertEquals(os.readlink(current_link), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_1, CLIENT_COMPONENT_NAME)) + + self.assertTrue(os.path.exists(os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + instance_manager.DEFAULT_MPACK_INSTANCE_NAME))) + + def test_create_mpack_server_module_with_default_component_instance(self): + create_mpack_with_defaults() + current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + instance_manager.DEFAULT_COMPONENT_INSTANCE_NAME, + instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertTrue(os.path.exists(current_link)) + self.assertEquals(os.readlink(current_link), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_1, SERVER_COMPONENT_NAME)) + + self.assertTrue(os.path.exists(os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + instance_manager.DEFAULT_MPACK_INSTANCE_NAME))) + + def test_create_mpack_server_module_with_multiple_component_instances(self): + create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME: ['server1', 'server2']}) + + current_link_1 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + 'server1', instance_manager.CURRENT_SOFTLINK_NAME) + self.assertTrue(os.path.exists(current_link_1)) + self.assertEqual(os.readlink(current_link_1), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_1, SERVER_COMPONENT_NAME)) + + current_link_2 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + 'server2', instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertTrue(os.path.exists(current_link_2)) + self.assertEqual(os.readlink(current_link_2), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_1, SERVER_COMPONENT_NAME)) + + def test_set_version_server_module_asterisk(self): + create_mpack_with_defaults() + + build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False) + + instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME, + SERVER_MODULE_NAME, '*') + + current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + instance_manager.DEFAULT_COMPONENT_INSTANCE_NAME, + instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertEqual(os.readlink(current_link), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_2, SERVER_COMPONENT_NAME)) + + def test_set_version_client_module_asterisk(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME) + + build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False) + + instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME, + CLIENT_MODULE_NAME, '*') + + current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, CLIENT_COMPONENT_NAME, + instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertEqual(os.readlink(current_link), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_2, CLIENT_COMPONENT_NAME)) + + def test_set_version_for_one_of_two_component_instances(self): + create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME: ['server1', 'server2']}) + + build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False) + + instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME, + SERVER_MODULE_NAME, None, {SERVER_COMPONENT_NAME: ['server2']}) + + current_link_1 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + 'server1', instance_manager.CURRENT_SOFTLINK_NAME) + self.assertEqual(os.readlink(current_link_1), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_1, SERVER_COMPONENT_NAME)) + + current_link_2 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME, MPACK_NAME, + INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME, + 'server2', instance_manager.CURRENT_SOFTLINK_NAME) + + self.assertEqual(os.readlink(current_link_2), + os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, MPACK_NAME, + MPACK_VERSION_2, SERVER_COMPONENT_NAME)) + + def test_get_conf_dir_all(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None, + components_map={SERVER_COMPONENT_NAME: ['server1']}) + + conf_dir_json = instance_manager.get_conf_dir() + + expected_json = { + "mpacks": { + "hdpcore": { + "mpack-instances": { + "Production": { + "name": "Production", + "subgroups": { + "default": { + "modules": { + "hdfs": { + "category": "SERVER", + "name": "hdfs", + "components": { + "hdfs_server": { + "component-instances": { + "server1": { + "config_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/hdfs_server/server1/conf", + "name": "server1" + } + } + } + } + }, + "hdfs_client": { + "category": "CLIENT", + "component_instances": { + "default": { + "config_dir": "/tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs_client/conf", + "name": "default" + } + }, + "name": "hdfs_client" + } + } + } + } + } + } + } + } + } + self.assertEqual(conf_dir_json, expected_json) + + def test_list_instances_all(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None, + components_map={SERVER_COMPONENT_NAME: ['server1']}) + + conf_dir_json = instance_manager.list_instances() + + expected_json = { + "mpacks": { + "hdpcore": { + "mpack-instances": { + "Production": { + "name": "Production", + "subgroups": { + "default": { + "modules": { + "hdfs": { + "category": "SERVER", + "name": "hdfs", + "components": { + "hdfs_server": { + "component-instances": { + "server1": { + "path": "/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1", + "name": "server1" + } + } + } + } + }, + "hdfs_client": { + "category": "CLIENT", + "component_instances": { + "default": { + "path": "/tmp/instance_manager_test/modules/hdfs-clients/3.1.0.0-b1", + "name": "default" + } + }, + "name": "hdfs_client" + } + } + } + } + } + } + } + } + } + self.assertEqual(conf_dir_json, expected_json) + + def test_granularity(self): + create_mpack_with_defaults() + + full_conf_dir_json = instance_manager.get_conf_dir() + self.assertTrue('mpacks' in full_conf_dir_json) + + mpack_conf_dir_json = instance_manager.get_conf_dir(mpack=MPACK_NAME) + self.assertTrue('mpack-instances' in mpack_conf_dir_json) + + instance_conf_dir_json = instance_manager.get_conf_dir(mpack=MPACK_NAME, mpack_instance=INSTANCE_NAME_1, + subgroup_name=None) + self.assertTrue('subgroups' in instance_conf_dir_json) + + subgroup_conf_dir_json = instance_manager.get_conf_dir(mpack=MPACK_NAME, mpack_instance=INSTANCE_NAME_1, + subgroup_name=SUBGROUP_NAME) + self.assertTrue('modules' in subgroup_conf_dir_json) + + module_conf_dir_json = instance_manager.get_conf_dir(mpack=MPACK_NAME, mpack_instance=INSTANCE_NAME_1, + subgroup_name=SUBGROUP_NAME, module_name=SERVER_MODULE_NAME) + self.assertTrue('components' in module_conf_dir_json) + + # The mpack level filter not specified + full_conf_dir_json = instance_manager.get_conf_dir(mpack_instance=INSTANCE_NAME_1, subgroup_name=SUBGROUP_NAME, + module_name=SERVER_MODULE_NAME) + self.assertTrue('mpacks' in full_conf_dir_json) + + # The instance level filter not specified + mpack_conf_dir_json = instance_manager.get_conf_dir(mpack=MPACK_NAME, subgroup_name=SUBGROUP_NAME, + module_name=SERVER_MODULE_NAME) + self.assertTrue('mpack-instances' in mpack_conf_dir_json) + + def test_filtering(self): + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None, + components_map={}) + + create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME, mpack_instance=INSTANCE_NAME_2) + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, mpack_instance=INSTANCE_NAME_2, components=None, + components_map={SERVER_COMPONENT_NAME: ['server1']}) + + build_rpm_structure(mpack_name=MPACK_NAME2, remove_old_content=False, create_modules=False) + create_mpack_with_defaults(mpack_name=MPACK_NAME2, mpack_instance=INSTANCE_NAME_2, + module_name=CLIENT_MODULE_NAME) + create_mpack_with_defaults(mpack_name=MPACK_NAME2, mpack_instance=INSTANCE_NAME_2, + module_name=SERVER_MODULE_NAME, components=None, + components_map={SERVER_COMPONENT_NAME: ['server2']}) + + filter_by_module_json = instance_manager.list_instances(module_name=SERVER_MODULE_NAME) + self.assertTrue(MPACK_NAME in filter_by_module_json['mpacks']) + self.assertTrue(MPACK_NAME2 in filter_by_module_json['mpacks']) + self.assertTrue(INSTANCE_NAME_2 in filter_by_module_json['mpacks'][MPACK_NAME]['mpack-instances']) + self.assertTrue(INSTANCE_NAME_1 not in filter_by_module_json['mpacks'][MPACK_NAME]['mpack-instances']) + + filter_by_component_instance_name_json = instance_manager.list_instances( + components_map={SERVER_COMPONENT_NAME: ['server2']}) + expected_filter_result = {'mpacks': {'edw': {'mpack-instances': {'eCommerce': {'name': 'eCommerce', 'subgroups': { + 'default': {'modules': {'hdfs': {'category': 'SERVER', 'name': 'hdfs', 'components': {'hdfs_server': { + 'component-instances': { + 'server2': {'path': '/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1', 'name': 'server2'}}}}}}}}}}}}} + self.assertEquals(expected_filter_result, filter_by_component_instance_name_json) + + def test_validation(self): + try: + create_mpack_with_defaults(mpack_name=MPACK_NAME2) + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, "Mpack {0} doesn't exist, please check mpack name.".format(MPACK_NAME2)) + + try: + create_mpack_with_defaults(mpack_version=MPACK_VERSION_2) + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "Mpack version {0} doesn't exist for mpack {1}, please check mpack name and version".format( + MPACK_VERSION_2, MPACK_NAME)) + + try: + create_mpack_with_defaults(module_name=SERVER_MODULE_NAME + "broken") + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "There is no module {0} for mpack {1} with version {2}." + " Please check mpack name, version and module name".format( + SERVER_MODULE_NAME + "broken", MPACK_NAME, MPACK_VERSION_1)) + + try: + create_mpack_with_defaults(components_map={SERVER_COMPONENT_NAME: "comp1"}, module_name=CLIENT_MODULE_NAME) + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "There is no component {0} in module {1} for mpack {2} with version {3}." + " Please check mpack name, version, module name and component name".format( + SERVER_COMPONENT_NAME, CLIENT_MODULE_NAME, MPACK_NAME, MPACK_VERSION_1)) + + def test_creating_existing_component_instance(self): + create_mpack_with_defaults() + try: + create_mpack_with_defaults() + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "The instance /tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/" + "hdfs_server/default already exist. To change the version use set-mpack-instance command") + + def test_set_non_existing_instance(self): + try: + instance_manager.set_mpack_instance(mpack=MPACK_NAME, mpack_version=MPACK_VERSION_1, + mpack_instance=INSTANCE_NAME_1, + subgroup_name=SUBGROUP_NAME, module_name=SERVER_MODULE_NAME, + components_map={}) + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "There are no created instances. Use create-mpack-instance command to add them.") + + create_mpack_with_defaults() + try: + instance_manager.set_mpack_instance(mpack=MPACK_NAME, mpack_version=MPACK_VERSION_1, + mpack_instance=INSTANCE_NAME_1, + subgroup_name=SUBGROUP_NAME, module_name=SERVER_MODULE_NAME, + components_map={SERVER_COMPONENT_NAME: ["non-existing-instance"]}) + raise AssertionError("The previous call should have thrown exception") + except ValueError as e: + self.assertEquals(e.message, + "Found no instances for the given filters.") + + +def create_mpack_with_defaults(mpack_name=MPACK_NAME, mpack_version=MPACK_VERSION_1, mpack_instance=INSTANCE_NAME_1, + subgroup_name=SUBGROUP_NAME, module_name=SERVER_MODULE_NAME, components='*', + components_map=None): + instance_manager.create_mpack(mpack_name, mpack_version, mpack_instance, + subgroup_name, module_name, components, components_map) + + +def build_rpm_structure(mpack_name=MPACK_NAME, mpack_version=MPACK_VERSION_1, mpack_json=MPACK_JSON, + module_version_mapping=MODULE_VERSION_MAPPING, + module_component_mapping=MODULE_COMPONENT_MAPPING, + remove_old_content=True, + create_modules=True): + if remove_old_content: + remove_rpm_structure() + + mpack_path = os.path.join(TMP_ROOT_FOLDER, instance_manager.MPACKS_FOLDER_NAME, mpack_name, mpack_version) + os.makedirs(mpack_path) + with open(os.path.join(mpack_path, "mpack.json"), "w") as mpack_json_file: + json.dump(mpack_json, mpack_json_file) + + modules_path = os.path.join(TMP_ROOT_FOLDER, instance_manager.MODULES_FOLDER_NAME) + for module_name in module_version_mapping: + if create_modules: + os.makedirs(os.path.join(modules_path, module_name, module_version_mapping[module_name], 'bin')) + + os.symlink(os.path.join(modules_path, module_name, module_version_mapping[module_name]), + os.path.join(mpack_path, module_component_mapping[module_name])) + + +def remove_rpm_structure(): + if os.path.exists(TMP_ROOT_FOLDER): + shutil.rmtree(TMP_ROOT_FOLDER) diff --git a/mpack-instance-manager/src/test/python/unitTests.py b/mpack-instance-manager/src/test/python/unitTests.py new file mode 100644 index 00000000000..b8ace6454b8 --- /dev/null +++ b/mpack-instance-manager/src/test/python/unitTests.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import unittest +import os +import sys +from random import shuffle +import fnmatch + +TEST_MASK = '[Tt]est*.py' +CUSTOM_TEST_MASK = '_[Tt]est*.py' + + +def get_parent_path(base, directory_name): + """ + Returns absolute path for directory_name, if directory_name present in base. + For example, base=/home/user/test2, directory_name=user - will return /home/user + """ + done = False + while not done: + base = os.path.dirname(base) + if base == "/": + return None + if os.path.split(base)[-1] == directory_name: + done = True + else: + done = False + return base + + +def get_test_files(path, mask=None, recursive=True): + """ + Returns test files for path recursively + """ + current = [] + directory_items = os.listdir(path) + directory_items.sort() + + for item in directory_items: + add_to_pythonpath = False + item_path = os.path.join(path, item) + if os.path.isfile(item_path): + if fnmatch.fnmatch(item, mask): + add_to_pythonpath = True + current.append(item) + elif os.path.isdir(item_path): + if recursive: + current.extend(get_test_files(item_path, mask=mask)) + if add_to_pythonpath: + sys.path.append(path) + return current + + +def main(): + custom_tests = False + if len(sys.argv) > 1: + if sys.argv[1] == "true": + custom_tests = True + pwd = os.path.abspath(os.path.dirname(__file__)) + + project_folder = get_parent_path(pwd, 'mpack-instance-manager') + sys.path.append(project_folder + "/src/test/python") + sys.path.append(project_folder + "/src/main/python") + sys.path.append(project_folder + "/src/main/python/instance_manager") + + has_failures = False + test_runs = 0 + test_failures = [] + test_errors = [] + # run mpack-instance-manager tests + sys.stderr.write("Running tests\n") + if custom_tests: + test_mask = CUSTOM_TEST_MASK + else: + test_mask = TEST_MASK + + tests = get_test_files(pwd, mask=test_mask, recursive=True) + shuffle(tests) + modules = [os.path.basename(s)[:-3] for s in tests] + suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in + modules] + testSuite = unittest.TestSuite(suites) + textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite) + test_runs += textRunner.testsRun + test_errors.extend([(str(item[0]), str(item[1]), "ERROR") for item in textRunner.errors]) + test_failures.extend([(str(item[0]), str(item[1]), "FAIL") for item in textRunner.failures]) + tests_status = textRunner.wasSuccessful() and not has_failures + + if not tests_status: + sys.stderr.write("----------------------------------------------------------------------\n") + sys.stderr.write("Failed tests:\n") + for failed_tests in [test_errors, test_failures]: + for err in failed_tests: + sys.stderr.write("{0}: {1}\n".format(err[2], err[0])) + sys.stderr.write("----------------------------------------------------------------------\n") + sys.stderr.write("{0}\n".format(err[1])) + sys.stderr.write("----------------------------------------------------------------------\n") + sys.stderr.write("Total run:{0}\n".format(test_runs)) + sys.stderr.write("Total errors:{0}\n".format(len(test_errors))) + sys.stderr.write("Total failures:{0}\n".format(len(test_failures))) + + if tests_status: + sys.stderr.write("OK\n") + exit_code = 0 + else: + sys.stderr.write("ERROR\n") + exit_code = 1 + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pom.xml b/pom.xml index 36df69adf45..a42704093c0 100644 --- a/pom.xml +++ b/pom.xml @@ -471,6 +471,7 @@ ambari-server ambari-funtest ambari-agent + mpack-instance-manager ambari-client ambari-shell ambari-logsearch @@ -495,6 +496,7 @@ ambari-server ambari-funtest ambari-agent + mpack-instance-manager ambari-client ambari-shell @@ -509,6 +511,7 @@ ambari-server ambari-funtest ambari-agent + mpack-instance-manager ambari-client ambari-shell ambari-logsearch @@ -532,6 +535,7 @@ ambari-server ambari-funtest ambari-agent + mpack-instance-manager ambari-client ambari-shell ambari-logsearch