From 226bd62f4c859f8f23eebfd26a07013af3874d35 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 3 Feb 2018 19:51:00 +0100 Subject: [PATCH 01/25] Move Setting-specific methods to Setting --- .../server/topology/BlueprintFactory.java | 11 +- .../ambari/server/topology/BlueprintImpl.java | 153 +---- .../server/topology/RepositorySetting.java | 22 +- .../ambari/server/topology/Setting.java | 138 +++- .../server/topology/SettingFactory.java | 13 +- .../BlueprintResourceProviderTest.java | 14 +- .../server/topology/BlueprintImplTest.java | 639 +----------------- .../server/topology/SettingFactoryTest.java | 26 +- .../ambari/server/topology/SettingTest.java | 181 +++-- .../server/topology/TopologyManagerTest.java | 3 +- 10 files changed, 306 insertions(+), 894 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 4386ec6919d..4fc5d67cddc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -40,7 +40,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -209,7 +208,7 @@ protected Stack createStack(StackId stackId) { //todo: Move logic to HostGroupImpl @SuppressWarnings("unchecked") private Collection processHostGroups(String bpName, StackDefinition stack, Map properties) { - Set> hostGroupProps = (HashSet>) + Set> hostGroupProps = (Set>) properties.get(HOST_GROUP_PROPERTY_ID); if (hostGroupProps == null || hostGroupProps.isEmpty()) { @@ -217,13 +216,13 @@ private Collection processHostGroups(String bpName, StackDefinition s } Collection hostGroups = new ArrayList<>(); - for (HashMap hostGroupProperties : hostGroupProps) { + for (Map hostGroupProperties : hostGroupProps) { String hostGroupName = (String) hostGroupProperties.get(HOST_GROUP_NAME_PROPERTY_ID); if (hostGroupName == null || hostGroupName.isEmpty()) { throw new IllegalArgumentException("Every host group must include a non-null 'name' property"); } - HashSet> componentProps = (HashSet>) + Set> componentProps = (Set>) hostGroupProperties.get(COMPONENT_PROPERTY_ID); Collection> configProps = (Collection>) @@ -240,7 +239,7 @@ private Collection processHostGroups(String bpName, StackDefinition s return hostGroups; } - private Collection processHostGroupComponents(StackDefinition stack, String groupName, HashSet> componentProps) { + private Collection processHostGroupComponents(StackDefinition stack, String groupName, Set> componentProps) { if (componentProps == null || componentProps.isEmpty()) { throw new IllegalArgumentException("Host group '" + groupName + "' must contain at least one component"); } @@ -248,7 +247,7 @@ private Collection processHostGroupComponents(StackDefinition stack, Collection stackComponentNames = getAllStackComponents(stack); Collection components = new ArrayList<>(); - for (HashMap componentProperties : componentProps) { + for (Map componentProperties : componentProps) { String componentName = componentProperties.get(COMPONENT_NAME_PROPERTY_ID); if (componentName == null || componentName.isEmpty()) { throw new IllegalArgumentException("Host group '" + groupName + diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 04616a72270..7e13d0d3099 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -46,11 +46,11 @@ import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.JsonUtils; -import org.apache.commons.lang.StringUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; /** @@ -83,7 +83,7 @@ public BlueprintImpl(BlueprintEntity entity, StackDefinition stack, Set hostGroups = parseBlueprintHostGroups(entity); // TODO: how to handle multiple stacks correctly? configuration.setParentConfiguration(stack.getConfiguration(getServices())); - setting = processSetting(entity.getSettings()); + setting = new Setting(parseSetting(entity.getSettings())); repoSettings = processRepoSettings(); } @@ -106,7 +106,7 @@ public BlueprintImpl(String name, Collection groups, StackDefinition if (configuration.getParentConfiguration() == null) { configuration.setParentConfiguration(stack.getConfiguration(getServices())); } - this.setting = setting; + this.setting = setting != null ? setting : new Setting(ImmutableMap.of()); repoSettings = processRepoSettings(); } @@ -178,115 +178,19 @@ public Collection getComponentNames(String service) { return getComponents(service).stream().map(Component::getName).collect(toList()); } - /** - * Get whether the specified component in the service is enabled - * for auto start. - * - * @param serviceName - Service name. - * @param componentName - Component name. - * - * @return null if value is not specified; true or false if specified. - */ @Override public String getRecoveryEnabled(String serviceName, String componentName) { - Set> settingValue; - - if (setting == null) - return null; - - // If component name was specified in the list of "component_settings", - // determine if recovery_enabled is true or false and return it. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_COMPONENT_SETTINGS); - for (Map setting : settingValue) { - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, componentName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - } - - // If component name is not specified, look up it's service. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS); - for ( Map setting : settingValue){ - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, serviceName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - } - - // If service name is not specified, look up the cluster setting. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); - for (Map setting : settingValue) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - - return null; + return setting.getRecoveryEnabled(serviceName, componentName); } - /** - * Get whether the specified service is enabled for credential store use. - * - * 
-   *     {@code
-   *       {
-   *         "service_settings" : [
-   *         { "name" : "RANGER",
-   *           "recovery_enabled" : "true",
-   *           "credential_store_enabled" : "true"
-   *         },
-   *         { "name" : "HIVE",
-   *           "recovery_enabled" : "true",
-   *           "credential_store_enabled" : "false"
-   *         },
-   *         { "name" : "TEZ",
-   *           "recovery_enabled" : "false"
-   *         }
-   *       ]
-   *     }
-   *   }
-   *
- * - * @param serviceName - Service name. - * - * @return null if value is not specified; true or false if specified. - */ @Override public String getCredentialStoreEnabled(String serviceName) { - if (setting == null) - return null; - - // Look up the service and return the credential_store_enabled value. - Set> settingValue = setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS); - for (Map setting : settingValue) { - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, serviceName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_CREDENTIAL_STORE_ENABLED))) { - return setting.get(Setting.SETTING_NAME_CREDENTIAL_STORE_ENABLED); - } - break; - } - } - - return null; + return setting.getCredentialStoreEnabled(serviceName); } @Override public boolean shouldSkipFailure() { - if (setting == null) { - return false; - } - Set> settingValue = setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS); - for (Map setting : settingValue) { - if (setting.containsKey(Setting.SETTING_NAME_SKIP_FAILURE)) { - return setting.get(Setting.SETTING_NAME_SKIP_FAILURE).equalsIgnoreCase("true"); - } - } - return false; + return setting.shouldSkipFailure(); } @Override @@ -405,15 +309,6 @@ private Configuration processConfiguration(Collection blueprintSetting) { - return blueprintSetting != null - ? new Setting(parseSetting(blueprintSetting)) - : null; - } - /** * Obtain configuration as a map of config type to corresponding properties. * @@ -436,14 +331,16 @@ private Map> parseConfigurations(Collection>> parseSetting(Collection blueprintSetting) { + private static Map>> parseSetting(Collection blueprintSetting) { + if (blueprintSetting == null) { + return ImmutableMap.of(); + } - Map>> properties = new HashMap<>(); + Map>> properties = new HashMap<>(); Gson gson = new Gson(); for (BlueprintSettingEntity setting : blueprintSetting) { String settingName = setting.getSettingName(); - Set> settingProperties = gson.>>fromJson( - setting.getSettingData(), Set.class); + Set> settingProperties = gson.>>fromJson(setting.getSettingData(), Set.class); properties.put(settingName, settingProperties); } return properties; @@ -606,9 +503,9 @@ private void createBlueprintSettingEntities(BlueprintEntity blueprintEntity) { Setting blueprintSetting = getSetting(); if (blueprintSetting != null) { Map settingEntityMap = new HashMap<>(); - for (Map.Entry>> propEntry : blueprintSetting.getProperties().entrySet()) { + for (Map.Entry>> propEntry : blueprintSetting.getProperties().entrySet()) { String settingName = propEntry.getKey(); - Set> properties = propEntry.getValue(); + Set> properties = propEntry.getValue(); BlueprintSettingEntity settingEntity = new BlueprintSettingEntity(); settingEntityMap.put(settingName, settingEntity); @@ -636,27 +533,7 @@ public boolean isValidConfigType(String configType) { * Parse stack repo info stored in the blueprint_settings table */ private List processRepoSettings() { - if (setting == null) { - return Collections.emptyList(); - } - - Set> repositorySettingsValue = setting.getSettingValue(Setting.SETTING_NAME_REPOSITORY_SETTINGS); - if (repositorySettingsValue == null) { - return Collections.emptyList(); - } - - return repositorySettingsValue.stream() - .map(this::parseRepositorySetting) - .collect(toList()); - } - - private RepositorySetting parseRepositorySetting(Map setting){ - RepositorySetting result = new RepositorySetting(); - result.setOperatingSystem(setting.get(RepositorySetting.OPERATING_SYSTEM)); - result.setOverrideStrategy(setting.get(RepositorySetting.OVERRIDE_STRATEGY)); - result.setRepoId(setting.get(RepositorySetting.REPO_ID)); - result.setBaseUrl(setting.get(RepositorySetting.BASE_URL)); - return result; + return setting != null ? setting.processRepoSettings() : Collections.emptyList(); } public List getRepositorySettings(){ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java index 35e370e83d8..3e7571c84d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.topology; +import java.util.Map; + public class RepositorySetting { /** * Settings for each repo setting sections @@ -34,6 +36,15 @@ public class RepositorySetting { private String repoId; private String baseUrl; + static RepositorySetting fromMap(Map setting) { + RepositorySetting result = new RepositorySetting(); + result.setOperatingSystem(setting.get(OPERATING_SYSTEM)); + result.setOverrideStrategy(setting.get(OVERRIDE_STRATEGY)); + result.setRepoId(setting.get(REPO_ID)); + result.setBaseUrl(setting.get(BASE_URL)); + return result; + } + /** * When specified under the "settings" section, it allows Ambari to overwrite existing repos stored * in the metainfo table in the Ambari server database. @@ -106,11 +117,10 @@ public void setBaseUrl(String baseUrl) { } public String toString(){ - StringBuilder strBldr = new StringBuilder(); - strBldr.append(OVERRIDE_STRATEGY);strBldr.append(": ");strBldr.append(overrideStrategy); - strBldr.append(OPERATING_SYSTEM);strBldr.append(": ");strBldr.append(operatingSystem); - strBldr.append(REPO_ID);strBldr.append(": ");strBldr.append(repoId); - strBldr.append(BASE_URL);strBldr.append(": ");strBldr.append(baseUrl); - return strBldr.toString(); + return + OVERRIDE_STRATEGY + ": " + overrideStrategy + + OPERATING_SYSTEM + ": " + operatingSystem + + REPO_ID + ": " + repoId + + BASE_URL + ": " + baseUrl; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java index 904c784dac3..3a09af119f0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java @@ -18,32 +18,31 @@ package org.apache.ambari.server.topology; -import java.util.Collections; -import java.util.HashMap; +import static java.util.stream.Collectors.toList; + +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.apache.commons.lang.StringUtils; + +import com.google.common.collect.ImmutableSet; + public class Setting { /** * Settings for this configuration instance */ - private Map>> properties; - - public static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings"; - - public static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings"; - - public static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings"; - - public static final String SETTING_NAME_DEPLOYMENT_SETTINGS = "deployment_settings"; - - public static final String SETTING_NAME_RECOVERY_ENABLED = "recovery_enabled"; + private final Map>> properties; + static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings"; + static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings"; + static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings"; + static final String SETTING_NAME_DEPLOYMENT_SETTINGS = "deployment_settings"; + static final String SETTING_NAME_RECOVERY_ENABLED = "recovery_enabled"; public static final String SETTING_NAME_SKIP_FAILURE = "skip_failure"; - - public static final String SETTING_NAME_NAME = "name"; - - public static final String SETTING_NAME_REPOSITORY_SETTINGS = "repository_settings"; + static final String SETTING_NAME_NAME = "name"; + static final String SETTING_NAME_REPOSITORY_SETTINGS = "repository_settings"; /** * When specified under the "service_settings" section, it indicates whether credential store @@ -64,15 +63,14 @@ public class Setting { * } * */ - public static final String SETTING_NAME_CREDENTIAL_STORE_ENABLED = "credential_store_enabled"; + private static final String SETTING_NAME_CREDENTIAL_STORE_ENABLED = "credential_store_enabled"; /** * Settings. * * @param properties setting name-->Set(property name-->property value) */ - public Setting(Map>> properties) { - + public Setting(Map>> properties) { this.properties = properties; } @@ -81,7 +79,7 @@ public Setting(Map>> properties) { * * @return map of properties for this settings instance keyed by setting name. */ - public Map>> getProperties() { + public Map>> getProperties() { return properties; } @@ -91,11 +89,101 @@ public Map>> getProperties() { * @param settingName * @return Set of Map of properties. */ - public Set> getSettingValue(String settingName) { - if (properties.containsKey(settingName)) { - return properties.get(settingName); + public Set> getSettingValue(String settingName) { + return properties.getOrDefault(settingName, ImmutableSet.of()); + } + + /** + * Get whether the specified service is enabled for credential store use. + * + * 
+   *     {@code
+   *       {
+   *         "service_settings" : [
+   *         { "name" : "RANGER",
+   *           "recovery_enabled" : "true",
+   *           "credential_store_enabled" : "true"
+   *         },
+   *         { "name" : "HIVE",
+   *           "recovery_enabled" : "true",
+   *           "credential_store_enabled" : "false"
+   *         },
+   *         { "name" : "TEZ",
+   *           "recovery_enabled" : "false"
+   *         }
+   *       ]
+   *     }
+   *   }
+   *
+ * + * @param serviceName - Service name. + * + * @return null if value is not specified; true or false if specified. + */ + String getCredentialStoreEnabled(String serviceName) { + return getStringFromNamedMap(SETTING_NAME_SERVICE_SETTINGS, serviceName, SETTING_NAME_CREDENTIAL_STORE_ENABLED); + } + + private String getStringFromNamedMap(String outerKey, String mapName, String innerKey) { + Set> maps = getSettingValue(outerKey); + for (Map map : maps) { + String name = map.get(SETTING_NAME_NAME); + if (Objects.equals(name, mapName)) { + String value = map.get(innerKey); + if (!StringUtils.isEmpty(value)) { + return value; + } + break; + } + } + return null; + } + + private String getString(String outerKey, String innerKey) { + Set> maps = getSettingValue(outerKey); + for (Map map : maps) { + if (map.containsKey(innerKey)) { + return map.get(innerKey); + } + } + return null; + } + + boolean shouldSkipFailure() { + return Boolean.parseBoolean(getString(SETTING_NAME_DEPLOYMENT_SETTINGS, SETTING_NAME_SKIP_FAILURE)); + } + + List processRepoSettings() { + Set> repositorySettingsValue = getSettingValue(SETTING_NAME_REPOSITORY_SETTINGS); + return repositorySettingsValue.stream() + .map(RepositorySetting::fromMap) + .collect(toList()); + } + + /** + * Get whether the specified component in the service is enabled + * for auto start. + * + * @param serviceName - Service name. + * @param componentName - Component name. + * + * @return null if value is not specified; true or false if specified. + */ + String getRecoveryEnabled(String serviceName, String componentName) { + Set> maps; + + // If component name was specified in the list of "component_settings", + // determine if recovery_enabled is true or false and return it. + String result = getStringFromNamedMap(SETTING_NAME_COMPONENT_SETTINGS, componentName, SETTING_NAME_RECOVERY_ENABLED); + // If component name is not specified, look up it's service. + if (result == null) { + result = getStringFromNamedMap(SETTING_NAME_SERVICE_SETTINGS, serviceName, SETTING_NAME_RECOVERY_ENABLED); + } + // If service name is not specified, look up the cluster setting. + if (result == null) { + result = getString(SETTING_NAME_RECOVERY_SETTINGS, SETTING_NAME_RECOVERY_ENABLED); } - return Collections.emptySet(); + return result; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java index a01f198ab1a..ff002821179 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java @@ -64,23 +64,20 @@ public class SettingFactory { /** * Attempts to build the list of settings in the following format: * setting_name1-->[propertyName1-->propertyValue1, propertyName2-->propertyValue2] - * @param blueprintSetting - * @return */ public static Setting getSetting(Collection> blueprintSetting) { - Map>> properties = new HashMap<>(); - Setting setting = new Setting(properties); + Map>> properties = new HashMap<>(); if (blueprintSetting != null) { for (Map settingMap : blueprintSetting) { for (Map.Entry entry : settingMap.entrySet()) { final String[] propertyNames = entry.getKey().split("/"); - Set> settingValue; + Set> settingValue; if (entry.getValue() instanceof Set) { - settingValue = (HashSet>)entry.getValue(); + settingValue = (Set>) entry.getValue(); } else if (propertyNames.length > 1){ - HashMap property = new HashMap<>(); + Map property = new HashMap<>(); property.put(propertyNames[1], String.valueOf(entry.getValue())); settingValue = properties.get(propertyNames[0]); if (settingValue == null) { @@ -96,6 +93,6 @@ else if (propertyNames.length > 1){ } } - return setting; + return new Setting(properties); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index dece002f9bb..5ee048d6dfc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -115,7 +115,7 @@ public static void initClass() { replay(resourceProviderFactory); } - private Map>> getSettingProperties() { + private Map>> getSettingProperties() { return new HashMap<>(); } @@ -133,7 +133,7 @@ public void testCreateResources() throws Exception { Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); @@ -208,7 +208,7 @@ public void testCreateResources_NoValidation() throws Exception { Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); requestInfoProperties.put("validate_topology", "false"); @@ -295,7 +295,7 @@ public void testCreateResources_withConfiguration() throws Exception { Map requestInfoProperties = getTestRequestInfoProperties(); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); @@ -369,7 +369,7 @@ public void testCreateResources_withSecurityConfiguration() throws Exception { Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null); // set expectations @@ -495,7 +495,7 @@ public void testCreateResources_withEmptyConfiguration() throws Exception { setConfigurationProperties(setProperties); AmbariManagementController managementController = createMock(AmbariManagementController.class); Map requestInfoProperties = new HashMap<>(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); requestInfoProperties.put(Request.REQUEST_INFO_BODY_PROPERTY, "{\"configurations\":[]}"); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); @@ -542,7 +542,7 @@ public void testCreateResources_withSingleConfigurationType() throws Exception { setConfigurationProperties(setProperties); AmbariManagementController managementController = createMock(AmbariManagementController.class); Map requestInfoProperties = new HashMap<>(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); requestInfoProperties.put(Request.REQUEST_INFO_BODY_PROPERTY, "{\"configurations\":[{\"configuration-type\":{\"properties\":{\"property\":\"value\"}}}]}"); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index 362f4e4889f..0019d48965d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -25,12 +25,10 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -42,17 +40,17 @@ import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ImmutableMap; + /** * Blueprint unit tests. */ public class BlueprintImplTest { private static final Map>> EMPTY_ATTRIBUTES = new HashMap<>(); - private static final Map> EMPTY_PROPERTIES = new HashMap<>(); - private static final Configuration EMPTY_CONFIGURATION = new Configuration(EMPTY_PROPERTIES, EMPTY_ATTRIBUTES); + private static final Configuration EMPTY_CONFIGURATION = Configuration.createEmpty(); Stack stack = createNiceMock(Stack.class); - Setting setting = createNiceMock(Setting.class); HostGroup group1 = createMock(HostGroup.class); HostGroup group2 = createMock(HostGroup.class); Set hostGroups = new HashSet<>(); @@ -304,9 +302,7 @@ public void testValidateConfigurations__secretReference() throws InvalidTopology public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws InvalidTopologyException, GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); + lzoProperties.put("core-site", ImmutableMap.of(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS)); Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); setupConfigurationWithGPLLicense(false); @@ -321,9 +317,7 @@ public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws I public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws InvalidTopologyException, GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); + lzoProperties.put("core-site", ImmutableMap.of(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS)); Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); setupConfigurationWithGPLLicense(false); @@ -338,10 +332,9 @@ public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws Inva public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException, GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); - put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); + lzoProperties.put("core-site", ImmutableMap.of( + BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS, + BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS)); Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); @@ -352,623 +345,9 @@ public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyExc verify(stack, group1, group2, serverConfig); } - @Test - public void testAutoSkipFailureEnabled() { - HashMap skipFailureSetting = new HashMap<>(); - skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); - expect(stack.getName()).andReturn("HDPCORE").anyTimes(); - expect(stack.getVersion()).andReturn("3.0.0.0").anyTimes(); - expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); - replay(stack, setting); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, setting); - assertTrue(blueprint.shouldSkipFailure()); - - verify(stack, setting); - } - - @Test - public void testAutoSkipFailureDisabled() { - HashMap skipFailureSetting = new HashMap<>(); - skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "false"); - expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); - replay(stack, setting); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, setting); - assertFalse(blueprint.shouldSkipFailure()); - - verify(stack, setting); - } - - private org.apache.ambari.server.configuration.Configuration setupConfigurationWithGPLLicense(boolean isGPLAllowed) { + private void setupConfigurationWithGPLLicense(boolean isGPLAllowed) { reset(serverConfig); expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce(); - return serverConfig; } - //todo: ensure coverage for these existing tests - - // private void validateEntity(BlueprintEntity entity, boolean containsConfig) { -// assertEquals(BLUEPRINT_NAME, entity.getBlueprintName()); -// -// StackEntity stackEntity = entity.getStack(); -// assertEquals("test-stack-name", stackEntity.getStackName()); -// assertEquals("test-stack-version", stackEntity.getStackVersion()); -// -// Collection hostGroupEntities = entity.getHostGroups(); -// -// assertEquals(2, hostGroupEntities.size()); -// for (HostGroupEntity hostGroup : hostGroupEntities) { -// assertEquals(BLUEPRINT_NAME, hostGroup.getBlueprintName()); -// assertNotNull(hostGroup.getBlueprintEntity()); -// Collection componentEntities = hostGroup.getComponents(); -// if (hostGroup.getName().equals("group1")) { -// assertEquals("1", hostGroup.getCardinality()); -// assertEquals(2, componentEntities.size()); -// Iterator componentIterator = componentEntities.iterator(); -// String name = componentIterator.next().getName(); -// assertTrue(name.equals("component1") || name.equals("component2")); -// String name2 = componentIterator.next().getName(); -// assertFalse(name.equals(name2)); -// assertTrue(name2.equals("component1") || name2.equals("component2")); -// } else if (hostGroup.getName().equals("group2")) { -// assertEquals("2", hostGroup.getCardinality()); -// assertEquals(1, componentEntities.size()); -// HostGroupComponentEntity componentEntity = componentEntities.iterator().next(); -// assertEquals("component1", componentEntity.getName()); -// -// if (containsConfig) { -// Collection configurations = hostGroup.getConfigurations(); -// assertEquals(1, configurations.size()); -// HostGroupConfigEntity hostGroupConfigEntity = configurations.iterator().next(); -// assertEquals(BLUEPRINT_NAME, hostGroupConfigEntity.getBlueprintName()); -// assertSame(hostGroup, hostGroupConfigEntity.getHostGroupEntity()); -// assertEquals("core-site", hostGroupConfigEntity.getType()); -// Map properties = gson.>fromJson( -// hostGroupConfigEntity.getConfigData(), Map.class); -// assertEquals(1, properties.size()); -// assertEquals("anything", properties.get("my.custom.hg.property")); -// } -// } else { -// fail("Unexpected host group name"); -// } -// } -// Collection configurations = entity.getConfigurations(); -// if (containsConfig) { -// assertEquals(1, configurations.size()); -// BlueprintConfigEntity blueprintConfigEntity = configurations.iterator().next(); -// assertEquals(BLUEPRINT_NAME, blueprintConfigEntity.getBlueprintName()); -// assertSame(entity, blueprintConfigEntity.getBlueprintEntity()); -// assertEquals("core-site", blueprintConfigEntity.getType()); -// Map properties = gson.>fromJson( -// blueprintConfigEntity.getConfigData(), Map.class); -// assertEquals(2, properties.size()); -// assertEquals("480", properties.get("fs.trash.interval")); -// assertEquals("8500", properties.get("ipc.client.idlethreshold")); -// } else { -// assertEquals(0, configurations.size()); -// } -// } - - - - // @Test -// public void testCreateResource_Validate__Cardinality__ExternalComponent() throws Exception { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// ((Set>) setProperties.iterator().next().get("configurations")). -// add(Collections.singletonMap("global/hive_database", "Existing MySQL Database")); -// -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("MYSQL_SERVER"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(blueprintFactory.createBlueprint(setProperties.iterator().next())).andReturn(blueprint).once(); -// expect(blueprintValidator.validateRequiredProperties()).andReturn(Collections.>>emptyMap()).once(blueprint); -// expect(blueprint.toEntity()).andReturn(entity); -// expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("MYSQL_SERVER").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "MYSQL_SERVER")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__MultipleDependencyInstances() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// DependencyInfo dependencyInfo = new DependencyInfo(); -// AutoDeployInfo autoDeployInfo = new AutoDeployInfo(); -// autoDeployInfo.setEnabled(false); -// dependencyInfo.setAutoDeploy(autoDeployInfo); -// dependencyInfo.setScope("cluster"); -// dependencyInfo.setName("test-service/component1"); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("component2").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component2")). -// andReturn(Collections.singletonList(dependencyInfo)).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__AutoCommit() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// // remove component2 from BP -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// DependencyInfo dependencyInfo = new DependencyInfo(); -// AutoDeployInfo autoDeployInfo = new AutoDeployInfo(); -// autoDeployInfo.setEnabled(true); -// autoDeployInfo.setCoLocate("test-service/component1"); -// dependencyInfo.setAutoDeploy(autoDeployInfo); -// dependencyInfo.setScope("cluster"); -// dependencyInfo.setName("test-service/component2"); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("component2").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component2")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.singletonList(dependencyInfo)).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__Fail() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("MYSQL_SERVER"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("MYSQL_SERVER").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "MYSQL_SERVER")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// try { -// provider.createResources(request); -// fail("Expected validation failure for MYSQL_SERVER"); -// } catch (IllegalArgumentException e) { -// // expected -// } -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__AmbariServerComponent() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException -// { -// Request request = createMock(Request.class); -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// -// Set> setProperties = getTestProperties(); -// ((HashSet>) ((HashSet>) setProperties.iterator().next().get( -// BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)).iterator().next().get("components")). -// iterator().next().put("name", "AMBARI_SERVER"); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.emptySet()); -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController); -// } - - } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java index f75e6c19d95..94c412d402d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java @@ -39,13 +39,9 @@ public class SettingFactoryTest { */ @Test public void testGetSettingWithSetOfProperties() { - SettingFactory settingFactory = new SettingFactory(); - Map>> properties; - - Setting setting = settingFactory.getSetting(createSettingWithSetOfProperties()); - Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); + Setting setting = SettingFactory.getSetting(createSettingWithSetOfProperties()); + Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); assertEquals(propertyValues.size(), 1); - assertEquals(propertyValues.iterator().next().get(Setting.SETTING_NAME_RECOVERY_ENABLED), "true"); } @@ -54,13 +50,9 @@ public void testGetSettingWithSetOfProperties() { */ @Test public void testGetSettingWithoutSetOfProperties() { - SettingFactory settingFactory = new SettingFactory(); - Map>> properties; - - Setting setting = settingFactory.getSetting(createSettingWithoutSetOfProperties()); - Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); + Setting setting = SettingFactory.getSetting(createSettingWithoutSetOfProperties()); + Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); assertEquals(propertyValues.size(), 1); - assertEquals(propertyValues.iterator().next().get(Setting.SETTING_NAME_RECOVERY_ENABLED), "true"); } @@ -72,13 +64,11 @@ public void testGetSettingWithoutSetOfProperties() { * } * ] * } - * - * @return */ private Collection> createSettingWithSetOfProperties() { - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); + Set> setting1 = new HashSet<>(); + Set> setting2 = new HashSet<>(); // Setting 1: Property1 HashMap setting1Properties1 = new HashMap<>(); @@ -117,8 +107,6 @@ private Collection> createSettingWithSetOfProperties() { * "recovery_enabled":"true" * } * } - * - * @return */ private Collection> createSettingWithoutSetOfProperties() { // Setting 2: Property1 and Property2 @@ -130,7 +118,7 @@ private Collection> createSettingWithoutSetOfProperties() { setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - Set> setting2 = new HashSet<>(); + Set> setting2 = new HashSet<>(); setting2.add(setting2Properties1); setting2.add(setting2Properties2); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java index 7ce3b38d13b..ecd5acbcd32 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java @@ -19,72 +19,129 @@ package org.apache.ambari.server.topology; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.junit.BeforeClass; import org.junit.Test; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + /** * Test the Setting class */ public class SettingTest { - /** - * Test get and set of entire setting. - */ - @Test - public void testGetProperties() { - Map>> properties = new HashMap<>(); - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); - Set> setting3 = new HashSet<>(); - Set> setting4 = new HashSet<>(); + + private static Setting setting; + private static Map>> properties; + private static Set> serviceSettings; + + @BeforeClass + public static void setup() { + properties = new HashMap<>(); + Set> recoverySettings = new HashSet<>(); + Set> deploymentSettings = new HashSet<>(); + Set> repositorySettings = new HashSet<>(); // Setting 1: Property1 - HashMap setting1Properties1 = new HashMap<>(); + Map setting1Properties1 = new HashMap<>(); setting1Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true"); - setting1.add(setting1Properties1); + recoverySettings.add(setting1Properties1); // Setting 2: Property1 and Property2 - HashMap setting2Properties1 = new HashMap<>(); - setting2Properties1.put(Setting.SETTING_NAME_NAME, "HDFS"); - setting2Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - - HashMap setting2Properties2 = new HashMap<>(); - setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); - setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - - setting2.add(setting2Properties1); - setting2.add(setting2Properties2); + Map hdfs = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "HDFS", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map yarn = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "YARN" + // no RECOVERY_ENABLED value for YARN + ); + + Map tez = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "TEZ", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + serviceSettings = ImmutableSet.of(hdfs, yarn, tez); + + Map hdfsClient = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "HDFS_CLIENT", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map namenode = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "NAMENODE", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + Map datanode = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "DATANODE" + // no RECOVERY_ENABLED value + ); + + Map yarnClient = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "YARN_CLIENT", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map resourceManager = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "RESOURCE_MANAGER", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + Map nodeManager = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "NODE_MANAGER" + // no RECOVERY_ENABLED value + ); + + Set> componentSettings = ImmutableSet.>builder() + .addAll(ImmutableSet.of(hdfsClient, namenode, datanode)) + .addAll(ImmutableSet.of(yarnClient, resourceManager, nodeManager)) + .build(); //Setting 3: Property 1 - HashMap setting3Properties1 = new HashMap<>(); - setting1Properties1.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); - setting1.add(setting3Properties1); + Map setting3Properties1 = new HashMap<>(); + setting3Properties1.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); + deploymentSettings.add(setting3Properties1); //Setting 4: Property 1 and 2 - HashMap setting4Properties1 = new HashMap<>(); + Map setting4Properties1 = new HashMap<>(); setting4Properties1.put(RepositorySetting.OVERRIDE_STRATEGY, RepositorySetting.OVERRIDE_STRATEGY_ALWAYS_APPLY); setting4Properties1.put(RepositorySetting.OPERATING_SYSTEM, "redhat7"); setting4Properties1.put(RepositorySetting.REPO_ID, "HDP"); setting4Properties1.put(RepositorySetting.BASE_URL, "http://localhost/repo"); - setting4.add(setting4Properties1); + repositorySettings.add(setting4Properties1); - HashMap setting4Properties2 = new HashMap<>(); + Map setting4Properties2 = new HashMap<>(); setting4Properties2.put(RepositorySetting.OVERRIDE_STRATEGY, RepositorySetting.OVERRIDE_STRATEGY_ALWAYS_APPLY); setting4Properties2.put(RepositorySetting.OPERATING_SYSTEM, "redhat7"); setting4Properties2.put(RepositorySetting.REPO_ID, "HDP-UTIL"); setting4Properties2.put(RepositorySetting.BASE_URL, "http://localhost/repo"); - setting4.add(setting4Properties2); + repositorySettings.add(setting4Properties2); + + properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, recoverySettings); + properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, serviceSettings); + properties.put(Setting.SETTING_NAME_COMPONENT_SETTINGS, componentSettings); + properties.put(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, deploymentSettings); + properties.put(Setting.SETTING_NAME_REPOSITORY_SETTINGS, repositorySettings); - properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, setting1); - properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, setting2); - properties.put(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, setting3); - properties.put(Setting.SETTING_NAME_REPOSITORY_SETTINGS, setting4); + setting = new Setting(properties); + } - Setting setting = new Setting(properties); + /** + * Test get and set of entire setting. + */ + @Test + public void testGetProperties() { assertEquals(properties, setting.getProperties()); } @@ -93,31 +150,47 @@ public void testGetProperties() { */ @Test public void testGetSettingProperties() { - Map>> properties = new HashMap<>(); - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); + assertEquals(serviceSettings, setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS)); + } - // Setting 1: Property1 - HashMap setting1Properties1 = new HashMap<>(); - setting1Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true"); - setting1.add(setting1Properties1); + @Test + public void recoveryEnabledAtComponentLevel() { + assertEquals("false", setting.getRecoveryEnabled("HDFS", "HDFS_CLIENT")); + assertEquals("true", setting.getRecoveryEnabled("HDFS", "NAMENODE")); + assertEquals("false", setting.getRecoveryEnabled("YARN", "YARN_CLIENT")); + assertEquals("true", setting.getRecoveryEnabled("YARN", "RESOURCE_MANAGER")); + } - // Setting 2: Property1 and Property2 - HashMap setting2Properties1 = new HashMap<>(); - setting2Properties1.put(Setting.SETTING_NAME_NAME, "HDFS"); - setting2Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); + @Test + public void recoveryEnabledAtServiceLevel() { + assertEquals("true", setting.getRecoveryEnabled("TEZ", "TEZ_CLIENT")); + assertEquals("false", setting.getRecoveryEnabled("HDFS", "DATANODE")); + } - HashMap setting2Properties2 = new HashMap<>(); - setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); - setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); + @Test + public void recoveryEnabledAtClusterLevel() { + assertEquals("true", setting.getRecoveryEnabled("OOZIE", "OOZIE_SERVER")); + assertEquals("true", setting.getRecoveryEnabled("YARN", "NODE_MANAGER")); + } - setting2.add(setting2Properties1); - setting2.add(setting2Properties2); + @Test + public void testAutoSkipFailureEnabled() { + Map skipFailureSetting = ImmutableMap.of(Setting.SETTING_NAME_SKIP_FAILURE, "true"); + Setting setting = new Setting(ImmutableMap.of(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, Collections.singleton(skipFailureSetting))); + assertTrue(setting.shouldSkipFailure()); + } - properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, setting1); - properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, setting2); + @Test + public void testAutoSkipFailureDisabled() { + Map skipFailureSetting = ImmutableMap.of(Setting.SETTING_NAME_SKIP_FAILURE, "false"); + Setting setting = new Setting(ImmutableMap.of(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, Collections.singleton(skipFailureSetting))); + assertFalse(setting.shouldSkipFailure()); + } - Setting setting = new Setting(properties); - assertEquals(setting2, setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS)); + @Test + public void testAutoSkipFailureUnspecified() { + Setting setting = new Setting(ImmutableMap.of()); + assertFalse(setting.shouldSkipFailure()); } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 7174a5e4f8e..31c4938a9a3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -41,6 +41,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -547,7 +548,7 @@ private void replayAll() { @Test(expected = InvalidTopologyException.class) public void testScaleHosts__alreadyExistingHost() throws InvalidTopologyTemplateException, InvalidTopologyException, AmbariException, NoSuchStackException { - HashSet> propertySet = new HashSet<>(); + Set> propertySet = new HashSet<>(); Map properties = new TreeMap<>(); properties.put(HostResourceProvider.HOST_HOST_NAME_PROPERTY_ID, "host1"); properties.put(HostResourceProvider.HOST_GROUP_PROPERTY_ID, "group1"); From 8f9d495eca81454e756a35985e437aa8bf257437 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 3 Feb 2018 20:46:19 +0100 Subject: [PATCH 02/25] Move HOSTGROUP_REGEX to BlueprintConfigurationProcessor --- .../internal/BlueprintConfigurationProcessor.java | 12 ++++++++---- .../server/topology/BlueprintValidatorImpl.java | 3 ++- .../ambari/server/topology/ClusterTopologyImpl.java | 5 +++-- .../org/apache/ambari/server/topology/HostGroup.java | 4 ---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 8da1f51ee2c..a167c098d07 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -49,7 +49,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.Configuration; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.validators.UnitValidatedProperty; import org.apache.commons.lang.StringUtils; @@ -71,6 +70,11 @@ public class BlueprintConfigurationProcessor { private static final Logger LOG = LoggerFactory.getLogger(BlueprintConfigurationProcessor.class); + /** + * Compiled regex for "%HOSTGROUP::...%" token. + */ + public static final Pattern HOST_GROUP_PLACEHOLDER_PATTERN = Pattern.compile("%HOSTGROUP::(\\S+?)%"); + private final static String COMMAND_RETRY_ENABLED_PROPERTY_NAME = "command_retry_enabled"; private final static String COMMANDS_TO_RETRY_PROPERTY_NAME = "commands_to_retry"; @@ -279,7 +283,7 @@ private boolean containsHostFromHostGroups(String configType, String propertyNam return false; } // check fir bp import - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(propertyValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(propertyValue); if (m.find()) { return true; } @@ -1381,7 +1385,7 @@ public String updateForClusterCreate(String propertyName, ClusterTopology topology) { //todo: getHostStrings - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(origValue); if (m.find()) { String hostGroupName = m.group(1); @@ -1406,7 +1410,7 @@ public Collection getRequiredHostGroups(String propertyName, Map> properties, ClusterTopology topology) { //todo: getHostStrings - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(origValue); if (m.find()) { String hostGroupName = m.group(1); return Collections.singleton(hostGroupName); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 39cbbfaf360..1915398bd00 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyConditionInfo; @@ -159,7 +160,7 @@ public void validateRequiredProperties(Blueprint blueprint) throws InvalidTopolo if(givenHostGroups.size() != hostGroupsForComponent.size()) { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } - if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ + if (BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()) { for (HostGroup hostGroupForComponent : hostGroupsForComponent) { Iterator itr = givenHostGroups.iterator(); while(itr.hasNext()){ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index f50e60f428d..b8a3b24b2c2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -31,6 +31,7 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.slf4j.Logger; @@ -215,8 +216,8 @@ private void validateTopology() } Map hadoopEnvConfig = configuration.getFullProperties().get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - if((!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"))) - || (!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))){ + if ((!BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"))) + || (!BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))) { throw new IllegalArgumentException("NAMENODE HA hosts mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected hosts are: " + nnHosts); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java index a329f42053f..848e2f891c8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -29,10 +29,6 @@ */ public interface HostGroup { - /** - * Compiled regex for hostgroup token. - */ - Pattern HOSTGROUP_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%"); /** * Get the name of the host group. * From 01aa3b374dcc48701e2edf2c6374e193b460f66b Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sat, 3 Feb 2018 20:49:13 +0100 Subject: [PATCH 03/25] Remove unused code from HostGroup --- .../ambari/server/topology/HostGroup.java | 11 +---------- .../ambari/server/topology/HostGroupImpl.java | 17 +---------------- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java index 848e2f891c8..24608059718 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -19,7 +19,6 @@ package org.apache.ambari.server.topology; import java.util.Collection; -import java.util.regex.Pattern; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.StackDefinition; @@ -43,14 +42,6 @@ public interface HostGroup { */ String getBlueprintName(); - /** - * Get the fully qualified host group name in the form of - * blueprintName:hostgroupName - * - * @return fully qualified host group name - */ - String getFullyQualifiedName(); - /** * Get all of the host group components. * @@ -88,7 +79,7 @@ public interface HostGroup { * @return set of component names */ @Deprecated - public Collection getComponentNames(String service); + Collection getComponentNames(String service); /** * Get the host group components which belong to the specified service. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java index 0899d0bf111..ffe6a249c63 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java @@ -35,8 +35,6 @@ import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.gson.Gson; @@ -45,8 +43,6 @@ */ public class HostGroupImpl implements HostGroup { - private static final Logger LOG = LoggerFactory.getLogger(HostGroupImpl.class); - /** * host group name */ @@ -73,7 +69,7 @@ public class HostGroupImpl implements HostGroup { */ private Configuration configuration = null; - private boolean containsMasterComponent = false; // FIXME never set + private boolean containsMasterComponent = false; private StackDefinition stack; @@ -110,17 +106,6 @@ public String getName() { return name; } - //todo: currently not qualifying host group name - @Override - public String getFullyQualifiedName() { - return String.format("%s:%s", blueprintName, getName()); - } - - //todo: currently not qualifying host group name - public static String formatAbsoluteName(String bpName, String hgName) { - return String.format("%s:%s", bpName, hgName); - } - @Override public Collection getComponents() { return components; From 1972f5db69af0427433e7f3592f86309b7e71d40 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2018 12:26:44 +0100 Subject: [PATCH 04/25] Move TopologyValidator into validators subpackage --- .../server/topology/validators/ChainedTopologyValidator.java | 1 - .../topology/validators/ClusterConfigTypeValidator.java | 1 - .../server/topology/validators/HiveServiceValidator.java | 1 - .../validators/RequiredConfigPropertiesValidator.java | 1 - .../topology/validators/RequiredPasswordValidator.java | 1 - .../server/topology/validators/StackConfigTypeValidator.java | 5 ----- .../server/topology/{ => validators}/TopologyValidator.java | 5 ++++- .../server/topology/validators/TopologyValidatorFactory.java | 2 -- .../ambari/server/topology/validators/UnitValidator.java | 1 - .../server/topology/ClusterDeployWithStartOnlyTest.java | 1 + .../ClusterInstallWithoutStartOnComponentLevelTest.java | 1 + .../server/topology/ClusterInstallWithoutStartTest.java | 1 + .../apache/ambari/server/topology/TopologyManagerTest.java | 1 + 13 files changed, 8 insertions(+), 14 deletions(-) rename ambari-server/src/main/java/org/apache/ambari/server/topology/{ => validators}/TopologyValidator.java (84%) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java index 8bcbcffbdc5..6815fd470ef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java @@ -32,7 +32,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java index 0170186d390..a6bdcc107d5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java @@ -18,7 +18,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index 9d9ad00a35b..bc9918ad8c5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -17,7 +17,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java index 4022fcba1e4..8a7402b3f53 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java @@ -27,7 +27,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index 5c9e0eb6555..9b537eada32 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -28,7 +28,6 @@ import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; /** * Validates that all required passwords are provided. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java index f028a31e1d0..4f51639e2d5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java @@ -19,7 +19,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,7 +57,3 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { } } } - - - - diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java similarity index 84% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java index c3e4b2bfa4c..41b4ad5561b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java @@ -16,7 +16,10 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; /** * Performs topology validation. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index bc76bff1e7b..ef44fcdb4ae 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -16,8 +16,6 @@ import java.util.List; -import org.apache.ambari.server.topology.TopologyValidator; - import com.google.common.collect.ImmutableList; public class TopologyValidatorFactory { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java index 31d5275d220..94f28f102e5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java @@ -27,7 +27,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; /** * I validate the unit of properties by checking if it matches to the stack defined unit. diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 55db7684f18..15dccb6b875 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -64,6 +64,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index bac0e5b9f56..9a7f6a2e855 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -65,6 +65,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index 3e11f5e8c64..a91d953e9e4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -65,6 +65,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 31c4938a9a3..61a45c8b411 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -70,6 +70,7 @@ import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMock; From 15b898a8b610e2bb5ff7804ff37e21bb5368a0a3 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2018 12:34:31 +0100 Subject: [PATCH 05/25] Fix warnings, typos, formatting --- .../validators/ChainedTopologyValidator.java | 6 +++--- .../validators/ClusterConfigTypeValidator.java | 2 +- .../RequiredConfigPropertiesValidator.java | 9 ++++----- .../validators/RequiredPasswordValidator.java | 12 ++---------- .../validators/TopologyValidatorFactory.java | 12 +++++++++--- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java index 6815fd470ef..0a10039b6a1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java @@ -36,14 +36,14 @@ import org.slf4j.LoggerFactory; /** - * Topology validator wrapper implementation. Executes a set of validations by calling a preconfgured set of validator implementations. + * Topology validator wrapper implementation. Executes a set of validations by calling a preconfigured set of validator implementations. */ public class ChainedTopologyValidator implements TopologyValidator { private static final Logger LOGGER = LoggerFactory.getLogger(ChainedTopologyValidator.class); - private List validators; + private final List validators; - public ChainedTopologyValidator(List validators) { + ChainedTopologyValidator(List validators) { this.validators = validators; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java index a6bdcc107d5..a65ea6dcde4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java @@ -33,7 +33,7 @@ public class ClusterConfigTypeValidator implements TopologyValidator { public void validate(ClusterTopology topology) throws InvalidTopologyException { // config types in from the request / configuration is always set in the request instance - Set topologyClusterConfigTypes = new HashSet(topology.getConfiguration().getAllConfigTypes()); + Set topologyClusterConfigTypes = new HashSet<>(topology.getConfiguration().getAllConfigTypes()); LOGGER.debug("Cluster config types: {}", topologyClusterConfigTypes); if (topologyClusterConfigTypes.isEmpty()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java index 8a7402b3f53..7b1a604d99e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java @@ -85,12 +85,11 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { for (String configType : requiredPropertiesByType.keySet()) { // We need a copy not to modify the original - Collection requiredPropertiesForType = new HashSet( - requiredPropertiesByType.get(configType)); + Collection requiredPropertiesForType = new HashSet<>(requiredPropertiesByType.get(configType)); if (!operationalConfigurations.containsKey(configType)) { // all required configuration is missing for the config type - missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); + missingProperties = addToMissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); continue; } @@ -99,7 +98,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { if (!requiredPropertiesForType.isEmpty()) { LOGGER.info("Found missing properties in hostgroup: {}, config type: {}, mising properties: {}", hostGroup.getName(), configType, requiredPropertiesForType); - missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); + missingProperties = addToMissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); } } } @@ -168,7 +167,7 @@ private Map>> getRequiredPropertiesByServ } - private Map> addTomissingProperties(Map> missingProperties, String hostGroup, Collection values) { + private Map> addToMissingProperties(Map> missingProperties, String hostGroup, Collection values) { Map> missing; if (missingProperties == null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index 9b537eada32..3db6948674c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -101,16 +101,8 @@ private Map>> validateRequiredPasswords(C String category = property.getType(); String name = property.getName(); if (! propertyExists(topology, groupProperties, category, name)) { - Map> missingHostGroupPropsMap = missingProperties.get(hostGroupName); - if (missingHostGroupPropsMap == null) { - missingHostGroupPropsMap = new HashMap<>(); - missingProperties.put(hostGroupName, missingHostGroupPropsMap); - } - Collection missingHostGroupTypeProps = missingHostGroupPropsMap.get(category); - if (missingHostGroupTypeProps == null) { - missingHostGroupTypeProps = new HashSet<>(); - missingHostGroupPropsMap.put(category, missingHostGroupTypeProps); - } + Map> missingHostGroupPropsMap = missingProperties.computeIfAbsent(hostGroupName, __ -> new HashMap<>()); + Collection missingHostGroupTypeProps = missingHostGroupPropsMap.computeIfAbsent(category, __ -> new HashSet<>()); missingHostGroupTypeProps.add(name); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index ef44fcdb4ae..983337a6f79 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -19,11 +19,17 @@ import com.google.common.collect.ImmutableList; public class TopologyValidatorFactory { - List validators; + + private final List validators; public TopologyValidatorFactory() { - validators = ImmutableList.of(new RequiredConfigPropertiesValidator(), new RequiredPasswordValidator(), new HiveServiceValidator(), - new StackConfigTypeValidator(), new UnitValidator(UnitValidatedProperty.ALL)); + validators = ImmutableList.of( + new RequiredConfigPropertiesValidator(), + new RequiredPasswordValidator(), + new HiveServiceValidator(), + new StackConfigTypeValidator(), + new UnitValidator(UnitValidatedProperty.ALL) + ); } public TopologyValidator createConfigurationValidatorChain() { From 936ed32ed064b5e8e0bc8daf520d312e4a06d0e8 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2018 15:58:09 +0100 Subject: [PATCH 06/25] Clean up StackConfigTypeValidatorTest --- .../StackConfigTypeValidatorTest.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java index 417c403e1fd..2292170fee7 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java @@ -18,8 +18,7 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.topology.Blueprint; @@ -55,8 +54,6 @@ public class StackConfigTypeValidatorTest extends EasyMockSupport { @Mock private ClusterTopology clusterTopologyMock; - private Set clusterRequestConfigTypes; - @TestSubject private StackConfigTypeValidator stackConfigTypeValidator = new StackConfigTypeValidator(); @@ -74,30 +71,24 @@ public void after() { } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenUnknownConfigTypeComesIn() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectInvalidType() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("invalid-site"))); - + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Collections.singleton("invalid-site")); replayAll(); // WHEN stackConfigTypeValidator.validate(clusterTopologyMock); - - // THEN - // exception is thrown - } @Test - public void testShouldValidationPassifNoConfigTypesomeIn() throws Exception { + public void allowEmptyConfig() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Collections.emptyList())); - + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Collections.emptySet()); replayAll(); // WHEN @@ -105,16 +96,27 @@ public void testShouldValidationPassifNoConfigTypesomeIn() throws Exception { // THEN // no exception is thrown - } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailIfMultipleInvalidConfigTypesComeIn() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectMultipleInvalidTypes() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("invalid-site-1", "invalid-default"))); + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("invalid-site-1", "invalid-default")); + replayAll(); + // WHEN + stackConfigTypeValidator.validate(clusterTopologyMock); + } + + @Test + public void allowValidTypes() throws InvalidTopologyException { + // GIVEN + List configTypes = Arrays.asList("core-site", "yarn-site"); + expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(configTypes); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(configTypes); replayAll(); // WHEN @@ -122,6 +124,5 @@ public void testShouldValidationFailIfMultipleInvalidConfigTypesComeIn() throws // THEN // no exception is thrown - } } From 04031e6cf87ac768982dc23046b9eb3eadb50b59 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2018 16:21:07 +0100 Subject: [PATCH 07/25] Verify that all stacks in cluster topology are known --- .../server/api/services/AmbariMetaInfo.java | 8 ++ .../server/topology/ClusterTopology.java | 7 ++ .../server/topology/ClusterTopologyImpl.java | 6 ++ .../validators/RejectUnknownStacks.java | 51 ++++++++++ .../validators/TopologyValidatorFactory.java | 23 +++-- .../validators/RejectUnknownStacksTest.java | 96 +++++++++++++++++++ 6 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java create mode 100644 ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 410903540cb..da2b2f3fad4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -725,6 +725,14 @@ public StackInfo getStack(StackId stackId) throws AmbariException { return getStack(stackId.getStackName(), stackId.getStackVersion()); } + public boolean isKnownStack(StackId stackId) { + try { + return getStack(stackId) != null; + } catch (AmbariException e) { + return false; + } + } + public StackInfo getStack(String stackName, String version) throws AmbariException { StackInfo stackInfoResult = stackManager.getStack(stackName, version); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index 69ccb617dae..dc89dd29fb8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -20,9 +20,11 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.state.StackId; /** * Represents a full cluster topology including all instance information as well as the associated @@ -51,6 +53,11 @@ public interface ClusterTopology { */ Blueprint getBlueprint(); + /** + * @return the set of stack (mpack) IDs associated with the blueprint + */ + Set getStackIds(); + /** * Get the cluster scoped configuration for the cluster. * This configuration has the blueprint cluster scoped diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index b8a3b24b2c2..f3f0fdfee63 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -34,6 +34,7 @@ import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.state.StackId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,6 +101,11 @@ public Blueprint getBlueprint() { return blueprint; } + @Override + public Set getStackIds() { + return blueprint.getStackIds(); + } + @Override public Configuration getConfiguration() { return configuration; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java new file mode 100644 index 00000000000..9821bc8582a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.joining; + +import javax.inject.Provider; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; + +/** + * Verifies that the topology only references known stacks. + */ +public class RejectUnknownStacks implements TopologyValidator { + + private final Provider metaInfo; + + RejectUnknownStacks(Provider metaInfo) { + this.metaInfo = metaInfo; + } + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + String unknownStacks = topology.getStackIds().stream() + .filter(stackId -> !metaInfo.get().isKnownStack(stackId)) + .sorted() + .map(Object::toString) + .collect(joining(", ")); + + if (!unknownStacks.isEmpty()) { + throw new InvalidTopologyException("Unknown stacks found in cluster creation request: " + unknownStacks); + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index 983337a6f79..de2c39d4152 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -16,20 +16,27 @@ import java.util.List; +import javax.inject.Inject; +import javax.inject.Provider; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; + import com.google.common.collect.ImmutableList; public class TopologyValidatorFactory { private final List validators; - public TopologyValidatorFactory() { - validators = ImmutableList.of( - new RequiredConfigPropertiesValidator(), - new RequiredPasswordValidator(), - new HiveServiceValidator(), - new StackConfigTypeValidator(), - new UnitValidator(UnitValidatedProperty.ALL) - ); + @Inject + public TopologyValidatorFactory(Provider metaInfo) { + validators = ImmutableList.builder() + .add(new RejectUnknownStacks(metaInfo)) + .add(new RequiredConfigPropertiesValidator()) + .add(new RequiredPasswordValidator()) + .add(new HiveServiceValidator()) + .add(new StackConfigTypeValidator()) + .add(new UnitValidator(UnitValidatedProperty.ALL)) + .build(); } public TopologyValidator createConfigurationValidatorChain() { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java new file mode 100644 index 00000000000..48cd6718308 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.easymock.EasyMockSupport; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; +import com.google.inject.util.Providers; + +public class RejectUnknownStacksTest extends EasyMockSupport { + + private final StackId validStackId = new StackId("valid", "1.0"); + private final StackId anotherValidStackId = new StackId("another", "2.1"); + private final StackId invalidStackId = new StackId("invalid", "3.2"); + private final StackId anotherInvalidStackId = new StackId("invalid", "1.1"); + private RejectUnknownStacks validator; + + @Before + public void setUp() { + AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class); + validator = new RejectUnknownStacks(Providers.of(metaInfo)); + + expect(metaInfo.isKnownStack(validStackId)).andReturn(true).anyTimes(); + expect(metaInfo.isKnownStack(anotherValidStackId)).andReturn(true).anyTimes(); + expect(metaInfo.isKnownStack(invalidStackId)).andReturn(false).anyTimes(); + expect(metaInfo.isKnownStack(anotherInvalidStackId)).andReturn(false).anyTimes(); + } + + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsUnknownStack() throws InvalidTopologyException { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(validStackId, invalidStackId)).anyTimes(); + replayAll(); + + // WHEN + validator.validate(topology); + } + + @Test + public void acceptsKnownStack() throws InvalidTopologyException { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(validStackId, anotherValidStackId)).anyTimes(); + replayAll(); + + // WHEN + validator.validate(topology); + + // THEN + // no exception + } + + @Test + public void reportsUnknownStacks() { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(invalidStackId, anotherInvalidStackId)).anyTimes(); + replayAll(); + + // WHEN + try { + validator.validate(topology); + fail("Expected " + InvalidTopologyException.class); + } catch (InvalidTopologyException e) { + // THEN + assertTrue(e.getMessage(), e.getMessage().contains(invalidStackId.toString())); + assertTrue(e.getMessage(), e.getMessage().contains(anotherInvalidStackId.toString())); + } + } +} From 0e71c77c7571da9545f4406507b3b11f7836b750 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2018 14:30:27 +0100 Subject: [PATCH 08/25] Add MpackInstance.getStackId --- .../org/apache/ambari/server/topology/BlueprintFactory.java | 2 +- .../org/apache/ambari/server/topology/MpackInstance.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 4fc5d67cddc..940661e9be2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -123,7 +123,7 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu mpackInstances = Collections.singleton(new MpackInstance(stackId.getStackName(), stackId.getStackVersion(), null, null, Configuration.createEmpty())); } Set stackIds = mpackInstances.stream() - .map(m -> new StackId(m.getMpackName(), m.getMpackVersion())) + .map(MpackInstance::getStackId) .collect(toSet()); StackDefinition stack = composeStacks(stackIds); Collection hostGroups = processHostGroups(name, stack, properties); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java index 6b57b0020e7..eeb45b776eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java @@ -26,6 +26,7 @@ import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity; import org.apache.ambari.server.orm.entities.BlueprintServiceConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintServiceEntity; +import org.apache.ambari.server.state.StackId; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -71,8 +72,8 @@ public void setMpackVersion(String mpackVersion) { } @JsonIgnore - public String getMpackNameAndVersion() { - return mpackName + "-" + mpackVersion; + public StackId getStackId() { + return new StackId(getMpackName(), getMpackVersion()); } @JsonIgnore From e87b3ccfa6c58afb44ee9f134b97e2a89001d6b3 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2018 14:36:04 +0100 Subject: [PATCH 09/25] Eliminate mpack-parser duplication in ProvisionClusterRequest --- .../internal/ProvisionClusterRequest.java | 43 +++++++++---------- .../server/topology/BlueprintFactory.java | 4 +- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index 115973c9ec5..99de97c9458 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -18,8 +18,8 @@ package org.apache.ambari.server.controller.internal; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -29,8 +29,10 @@ import org.apache.ambari.server.api.predicate.InvalidQueryException; import org.apache.ambari.server.security.encryption.CredentialStoreType; import org.apache.ambari.server.stack.NoSuchStackException; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileBuilder; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileEvaluationException; +import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.ConfigurationFactory; @@ -44,12 +46,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Enums; import com.google.common.base.Optional; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; /** * Request for provisioning a cluster. @@ -129,8 +130,6 @@ public class ProvisionClusterRequest extends BaseClusterRequest { public static final String MANAGEMENT_PACK_MAPPINGS_PROPERTY = "management_pack_mappings"; - public static final String MPACK_INSTANCES_PROPERTY = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID; - public static final String MPACK_INSTANCE_PROPERTY = "mpack_instance"; public static final String COMPONENT_NAME_PROPERTY = "component_name"; @@ -163,7 +162,9 @@ public class ProvisionClusterRequest extends BaseClusterRequest { private final String quickLinksProfileJson; - private Collection mpackInstances; + private final Collection mpackInstances; + private final Set stackIds; + private final StackDefinition stack; private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class); @@ -213,7 +214,10 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura setProvisionAction(parseProvisionAction(properties)); - processMpackInstances(properties); + mpackInstances = BlueprintFactory.createMpackInstances(properties); + Set stackIdsInRequest = mpackInstances.stream().map(MpackInstance::getStackId).collect(toSet()); // FIXME persist these + stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), stackIdsInRequest)); + stack = blueprintFactory.composeStacks(stackIds); try { this.quickLinksProfileJson = processQuickLinksProfile(properties); @@ -222,21 +226,6 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura } } - private void processMpackInstances(Map properties) throws InvalidTopologyTemplateException { - if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) { - ObjectMapper mapper = new ObjectMapper(); - mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - try { - String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY)); - this.mpackInstances = mapper.readValue(mpackInstancesJson, - new TypeReference>() {}); - } - catch (IOException ex) { - throw new InvalidTopologyTemplateException("Cannot process mpack instances.", ex); - } - } - } - private String processQuickLinksProfile(Map properties) throws QuickLinksProfileEvaluationException { Object globalFilters = properties.get(QUICKLINKS_PROFILE_FILTERS_PROPERTY); Object serviceFilters = properties.get(QUICKLINKS_PROFILE_SERVICES_PROPERTY); @@ -534,6 +523,14 @@ public String getDefaultPassword() { return defaultPassword; } + public Set getStackIds() { + return stackIds; + } + + public StackDefinition getStack() { + return stack; + } + public Collection getMpackInstances() { return mpackInstances; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 940661e9be2..07ecce29180 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -134,7 +134,7 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu return new BlueprintImpl(name, hostGroups, stack, stackIds, mpackInstances, configuration, securityConfiguration, setting); } - private Collection createMpackInstances(Map properties) throws NoSuchStackException { + public static Collection createMpackInstances(Map properties) throws NoSuchStackException { if (properties.containsKey(MPACK_INSTANCES_PROPERTY_ID)) { ObjectMapper mapper = new ObjectMapper(); mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); @@ -156,7 +156,7 @@ private static StackId getStackId(Map properties) throws NoSuchS return new StackId(stackName, stackVersion); } - private StackDefinition composeStacks(Set stackIds) { + public StackDefinition composeStacks(Set stackIds) { Set stacks = stackIds.stream() .map(this::createStack) .collect(toSet()); From 1dcfd709de7cb8445a69eed0996bd10b6b40abdb Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2018 14:34:47 +0100 Subject: [PATCH 10/25] Promote StackFactory to top-level --- .../server/api/services/AmbariMetaInfo.java | 7 +-- .../server/controller/ControllerModule.java | 3 + .../internal/ExportBlueprintRequest.java | 18 +++--- .../server/controller/internal/Stack.java | 8 +-- .../server/topology/BlueprintFactory.java | 31 +--------- .../server/topology/DefaultStackFactory.java | 43 ++++++++++++++ .../ambari/server/topology/StackFactory.java | 32 ++++++++++ .../internal/ExportBlueprintRequestTest.java | 3 +- .../server/topology/BlueprintFactoryTest.java | 58 +++++-------------- 9 files changed, 111 insertions(+), 92 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index da2b2f3fad4..baf80f37d35 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -719,21 +719,20 @@ public Collection getStacks(String stackName) throws AmbariException * @param stackId * the stack id * @return the stack info - * @throws AmbariException */ - public StackInfo getStack(StackId stackId) throws AmbariException { + public StackInfo getStack(StackId stackId) throws StackAccessException { return getStack(stackId.getStackName(), stackId.getStackVersion()); } public boolean isKnownStack(StackId stackId) { try { return getStack(stackId) != null; - } catch (AmbariException e) { + } catch (StackAccessException e) { return false; } } - public StackInfo getStack(String stackName, String version) throws AmbariException { + public StackInfo getStack(String stackName, String version) throws StackAccessException { StackInfo stackInfoResult = stackManager.getStack(stackName, version); if (stackInfoResult == null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index 357f1b61d86..5cfea7c2aca 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -171,9 +171,11 @@ import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.BlueprintValidator; import org.apache.ambari.server.topology.BlueprintValidatorImpl; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; import org.apache.ambari.server.topology.PersistedStateImpl; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; import org.apache.ambari.server.utils.PasswordUtils; import org.apache.ambari.server.view.ViewInstanceHandlerList; @@ -542,6 +544,7 @@ private void installFactories() { bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class); bind(SecurityHelper.class).toInstance(SecurityHelperImpl.getInstance()); bind(BlueprintValidator.class).to(BlueprintValidatorImpl.class); + bind(StackFactory.class).to(DefaultStackFactory.class); bind(BlueprintFactory.class); install(new FactoryModuleBuilder().implement(AmbariEvent.class, Names.named("userCreated"), UserCreatedEvent.class).build(AmbariEventFactory.class)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java index 13ae732cec5..ac30db97040 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; -import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.api.util.TreeNode; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; @@ -83,7 +83,7 @@ public ExportBlueprintRequest(TreeNode clusterNode) throws InvalidTopo Collection exportedHostGroups = processHostGroups(clusterNode.getChild("hosts")); createHostGroupInfo(exportedHostGroups); - createBlueprint(exportedHostGroups, parseStack(clusterResource)); + createBlueprint(exportedHostGroups, createStack(parseStack(clusterResource))); } public String getClusterName() { @@ -153,15 +153,15 @@ private void createHostGroupInfo(Collection exportedHostGroup } - private Stack parseStack(Resource clusterResource) throws InvalidTopologyTemplateException { - String[] stackTokens = String.valueOf(clusterResource.getPropertyValue( - ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID)).split("-"); + private StackId parseStack(Resource clusterResource) { + return new StackId(String.valueOf(clusterResource.getPropertyValue(ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID))); + } + private Stack createStack(StackId stackId) throws InvalidTopologyTemplateException { try { - return new Stack(stackTokens[0], stackTokens[1], controller); - } catch (AmbariException e) { - throw new InvalidTopologyTemplateException(String.format( - "The specified stack doesn't exist: name=%s version=%s", stackTokens[0], stackTokens[1])); + return new Stack(stackId, controller.getAmbariMetaInfo()); + } catch (StackAccessException e) { + throw new InvalidTopologyTemplateException(String.format("The specified stack doesn't exist: %s", stackId)); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index 77422eb2fa4..5f3186bdc54 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -26,8 +26,8 @@ import java.util.Set; import java.util.stream.Stream; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.ReadOnlyConfigurationResponse; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; @@ -127,8 +127,8 @@ public class Stack implements StackDefinition { private Map> excludedConfigurationTypes = new HashMap<>(); - public Stack(String name, String version, AmbariManagementController ctrl) throws AmbariException { // FIXME remove or at least change to use metainfo directly - this(ctrl.getAmbariMetaInfo().getStack(name, version)); + public Stack(StackId stackId, AmbariMetaInfo metaInfo) throws StackAccessException { + this(metaInfo.getStack(stackId)); } public Stack(StackInfo stackInfo) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 07ecce29180..84ea5129be7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -47,10 +47,7 @@ import java.util.function.Function; import java.util.stream.Stream; -import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ObjectNotFoundException; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.Stack; @@ -195,13 +192,9 @@ static void verifyStackDefinitionsAreDisjoint(Stream items, String type, protected Stack createStack(StackId stackId) { try { //todo: don't pass in controller - return stackFactory.createStack(stackId, AmbariServer.getController()); + return stackFactory.createStack(stackId); } catch (ObjectNotFoundException e) { throw new NoSuchStackException(stackId); - } catch (AmbariException e) { - // todo - throw new RuntimeException( - String.format("An error occurred parsing the stack information for %s", stackId) , e); } } @@ -296,26 +289,4 @@ public static void init(BlueprintDAO dao) { blueprintDAO = dao; } - /** - * Internal interface used to abstract out the process of creating the Stack object. - * - * This is used to simplify unit testing, since a new Factory can be provided to - * simulate various Stack or error conditions. - */ - interface StackFactory { - Stack createStack(StackId stackId, AmbariManagementController managementController) throws AmbariException; - } - - /** - * Default implementation of StackFactory. - * - * Calls the Stack constructor to create the Stack instance. - * - */ - private static class DefaultStackFactory implements StackFactory { - @Override - public Stack createStack(StackId stackId, AmbariManagementController managementController) throws AmbariException { - return new Stack(stackId.getStackName(), stackId.getStackVersion(), managementController); - } - } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java new file mode 100644 index 00000000000..2eb1bd64aa9 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import javax.inject.Inject; +import javax.inject.Provider; + +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.StackId; + +/** + * Default implementation of StackFactory. + * + * Calls the Stack constructor to create the Stack instance. + * + */ +public class DefaultStackFactory implements StackFactory { + + @Inject + private Provider metaInfo; + + @Override + public Stack createStack(StackId stackId) throws StackAccessException { + return new Stack(stackId, metaInfo.get()); + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java new file mode 100644 index 00000000000..d7b07401603 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.StackId; + +/** + * Internal interface used to abstract out the process of creating the Stack object. + * + * This is used to simplify unit testing, since a new Factory can be provided to + * simulate various Stack or error conditions. + */ +public interface StackFactory { + Stack createStack(StackId stackId) throws StackAccessException; +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java index 65d89c11d2f..0ab9dfb612c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java @@ -44,6 +44,7 @@ import org.apache.ambari.server.api.util.TreeNodeImpl; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.HostGroup; @@ -77,7 +78,7 @@ public void setupTest() throws Exception { AmbariMetaInfo metainfo = createNiceMock(AmbariMetaInfo.class); expect(controller.getAmbariMetaInfo()).andReturn(metainfo).anyTimes(); StackInfo stackInfo = createNiceMock(StackInfo.class); - expect(metainfo.getStack("TEST", "1.0")).andReturn(stackInfo); + expect(metainfo.getStack(new StackId("TEST", "1.0"))).andReturn(stackInfo); expect(stackInfo.getServices()).andReturn(Collections.emptySet()).anyTimes(); expect(stackInfo.getProperties()).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index 722345fc0fa..d347fd907df 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -19,16 +19,13 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toSet; -import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.powermock.api.easymock.PowerMock.createStrictMock; -import static org.powermock.api.easymock.PowerMock.expectNew; import static org.powermock.api.easymock.PowerMock.replay; import static org.powermock.api.easymock.PowerMock.reset; import static org.powermock.api.easymock.PowerMock.verify; @@ -40,8 +37,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.ambari.server.ObjectNotFoundException; -import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider; import org.apache.ambari.server.controller.internal.BlueprintResourceProviderTest; import org.apache.ambari.server.controller.internal.Stack; @@ -50,7 +46,6 @@ import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.StackId; -import org.easymock.EasyMockSupport; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -71,13 +66,13 @@ public class BlueprintFactoryTest { private static final String BLUEPRINT_NAME = "test-blueprint"; - BlueprintFactory factory = new BlueprintFactory(); - Stack stack = createNiceMock(Stack.class); - BlueprintFactory testFactory = new TestBlueprintFactory(stack); - BlueprintDAO dao = createStrictMock(BlueprintDAO.class); - BlueprintEntity entity = createStrictMock(BlueprintEntity.class); - BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); - + private BlueprintFactory factory = new BlueprintFactory(); + private Stack stack = createNiceMock(Stack.class); + private BlueprintFactory testFactory = new TestBlueprintFactory(stack); + private BlueprintDAO dao = createStrictMock(BlueprintDAO.class); + private BlueprintEntity entity = createStrictMock(BlueprintEntity.class); + private BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); + private StackFactory stackFactory = createNiceMock(StackFactory.class); @Before public void init() throws Exception { @@ -102,24 +97,6 @@ public void tearDown() { reset(stack, dao, entity, configEntity); } - //todo: implement -// @Test -// public void testGetBlueprint() throws Exception { -// -// Collection configs = new ArrayList(); -// configs.add(configEntity); -// -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(entity).once(); -// expect(entity.getBlueprintName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); -// expect(entity.getConfigurations()).andReturn(configs).atLeastOnce(); -// -// replay(dao, entity); -// -// Blueprint blueprint = factory.getBlueprint(BLUEPRINT_NAME); -// -// -// } - @Test public void testGetMultiInstanceBlueprint() throws Exception { // prepare @@ -135,9 +112,9 @@ public void testGetMultiInstanceBlueprint() throws Exception { Stack edwStack = createNiceMock(Stack.class); expect(edwStack.getName()).andReturn(edw.getStackName()).anyTimes(); expect(edwStack.getVersion()).andReturn(edw.getStackVersion()).anyTimes(); - expectNew(Stack.class, eq(hdp.getStackName()), eq(hdp.getStackVersion()), anyObject(AmbariManagementController.class)).andReturn(hdpStack).anyTimes(); - expectNew(Stack.class, eq(edw.getStackVersion()), eq(edw.getStackVersion()), anyObject(AmbariManagementController.class)).andReturn(edwStack).anyTimes(); - replay(Stack.class, hdpStack, edwStack, dao); + expect(stackFactory.createStack(hdp)).andReturn(hdpStack).anyTimes(); + expect(stackFactory.createStack(edw)).andReturn(edwStack).anyTimes(); + replay(hdpStack, edwStack, dao, stackFactory); // test Blueprint blueprint = testFactory.getBlueprint(BLUEPRINT_NAME); @@ -253,20 +230,13 @@ public Blueprint createMultiInstanceBlueprint() throws Exception { @Test(expected=NoSuchStackException.class) public void testCreateInvalidStack() throws Exception { - EasyMockSupport mockSupport = new EasyMockSupport(); - BlueprintFactory.StackFactory mockStackFactory = - mockSupport.createMock(BlueprintFactory.StackFactory.class); - // setup mock to throw exception, to simulate invalid stack request - expect(mockStackFactory.createStack(new StackId(), null)).andThrow(new ObjectNotFoundException("Invalid Stack")); - - mockSupport.replayAll(); + expect(stackFactory.createStack(new StackId())).andThrow(new StackAccessException("Invalid Stack")); + replay(stackFactory); BlueprintFactory factoryUnderTest = - new BlueprintFactory(mockStackFactory); + new BlueprintFactory(stackFactory); factoryUnderTest.createStack(new StackId()); - - mockSupport.verifyAll(); } @Test(expected=IllegalArgumentException.class) From 51147a0eb6db795b36ac483931650efb3b228558 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2018 18:21:30 +0100 Subject: [PATCH 11/25] Eliminate static injection in BlueprintFactory --- .../server/controller/AmbariServer.java | 2 - .../server/topology/BlueprintFactory.java | 31 +++------ .../server/topology/BlueprintFactoryTest.java | 64 ++++--------------- 3 files changed, 21 insertions(+), 76 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index e93277e541f..fbe80ed0226 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -81,7 +81,6 @@ import org.apache.ambari.server.metrics.system.MetricsService; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.PersistenceType; -import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.GroupDAO; import org.apache.ambari.server.orm.dao.MetainfoDAO; @@ -948,7 +947,6 @@ public void performStaticInjection() { injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory .class), injector.getInstance(Gson.class)); HostResourceProvider.setTopologyManager(injector.getInstance(TopologyManager.class)); - BlueprintFactory.init(injector.getInstance(BlueprintDAO.class)); BaseClusterRequest.init(injector.getInstance(BlueprintFactory.class)); AmbariContext.init(injector.getInstance(HostRoleCommandFactory.class)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 84ea5129be7..9ae3a3c8cc4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -47,6 +47,9 @@ import java.util.function.Function; import java.util.stream.Stream; +import javax.inject.Inject; +import javax.inject.Provider; + import org.apache.ambari.server.ObjectNotFoundException; import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.internal.ProvisionAction; @@ -64,7 +67,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; -import com.google.inject.Inject; /** * Create a Blueprint instance. @@ -73,21 +75,18 @@ public class BlueprintFactory { private static final Logger LOG = LoggerFactory.getLogger(BlueprintFactory.class); - private static BlueprintDAO blueprintDAO; - + private final Provider blueprintDAO; private final ConfigurationFactory configFactory = new ConfigurationFactory(); private final StackFactory stackFactory; - public BlueprintFactory() { - this(new DefaultStackFactory()); - } - - protected BlueprintFactory(StackFactory stackFactory) { + @Inject + public BlueprintFactory(Provider blueprintDAO, StackFactory stackFactory) { + this.blueprintDAO = blueprintDAO; this.stackFactory = stackFactory; } public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException { - BlueprintEntity entity = blueprintDAO.findByName(blueprintName); + BlueprintEntity entity = blueprintDAO.get().findByName(blueprintName); if (entity != null) { Set stackIds = entity.getMpackInstances().stream() .map(m -> new StackId(m.getMpackName(), m.getMpackVersion())) @@ -189,9 +188,8 @@ static void verifyStackDefinitionsAreDisjoint(Stream items, String type, } } - protected Stack createStack(StackId stackId) { + private Stack createStack(StackId stackId) { try { - //todo: don't pass in controller return stackFactory.createStack(stackId); } catch (ObjectNotFoundException e) { throw new NoSuchStackException(stackId); @@ -278,15 +276,4 @@ private Collection getAllStackComponents(StackDefinition stack) { return allComponents; } - - /** - * Static initialization. - * - * @param dao blueprint data access object - */ - @Inject - public static void init(BlueprintDAO dao) { - blueprintDAO = dao; - } - } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index d347fd907df..053cc0522ab 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -30,54 +30,43 @@ import static org.powermock.api.easymock.PowerMock.reset; import static org.powermock.api.easymock.PowerMock.verify; -import java.lang.reflect.Field; import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider; import org.apache.ambari.server.controller.internal.BlueprintResourceProviderTest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintEntity; -import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.StackId; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.inject.util.Providers; /** * BlueprintFactory unit tests. */ -@SuppressWarnings("unchecked") -@RunWith(PowerMockRunner.class) -@PrepareForTest(BlueprintImpl.class) public class BlueprintFactoryTest { private static final String BLUEPRINT_NAME = "test-blueprint"; - private BlueprintFactory factory = new BlueprintFactory(); private Stack stack = createNiceMock(Stack.class); - private BlueprintFactory testFactory = new TestBlueprintFactory(stack); private BlueprintDAO dao = createStrictMock(BlueprintDAO.class); private BlueprintEntity entity = createStrictMock(BlueprintEntity.class); private BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); - private StackFactory stackFactory = createNiceMock(StackFactory.class); + private BlueprintFactory testFactory = new BlueprintFactory(Providers.of(dao), __ -> stack); @Before public void init() throws Exception { - setPrivateField(factory, "blueprintDAO", dao); - Set stackIds = ImmutableSet.of(new StackId("stack", "0.1")); Collection services = ImmutableSet.of("test-service1", "test-service2"); Collection components = ImmutableSet.of("component1", "component2"); @@ -112,22 +101,23 @@ public void testGetMultiInstanceBlueprint() throws Exception { Stack edwStack = createNiceMock(Stack.class); expect(edwStack.getName()).andReturn(edw.getStackName()).anyTimes(); expect(edwStack.getVersion()).andReturn(edw.getStackVersion()).anyTimes(); - expect(stackFactory.createStack(hdp)).andReturn(hdpStack).anyTimes(); - expect(stackFactory.createStack(edw)).andReturn(edwStack).anyTimes(); - replay(hdpStack, edwStack, dao, stackFactory); + replay(hdpStack, edwStack, dao); // test Blueprint blueprint = testFactory.getBlueprint(BLUEPRINT_NAME); Set mpackNames = blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(Collectors.toSet()); assertEquals(ImmutableSet.of("HDPCORE-3.0", "EDW-3.1"), mpackNames ); - MpackInstance hdpCore = - blueprint.getMpacks().stream().filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())).findAny().get(); + Optional hdpCore = blueprint.getMpacks().stream() + .filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())) + .findAny(); + assertTrue(hdpCore.isPresent()); + MpackInstance mpackInstance = hdpCore.get(); Set serviceInstanceNames = - hdpCore.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet()); + mpackInstance.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet()); assertEquals(ImmutableSet.of("ZK1", "ZK2"), serviceInstanceNames); Set serviceInstanceTypes = - hdpCore.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet()); + mpackInstance.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet()); assertEquals(ImmutableSet.of("ZOOKEEPER"), serviceInstanceTypes); Set stackIds = blueprint.getStackIds(); assertEquals(ImmutableSet.of(hdp, edw), stackIds); @@ -139,6 +129,7 @@ public void testGetBlueprint_NotFound() throws Exception { expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null).once(); replay(dao, entity, configEntity); + BlueprintFactory factory = new BlueprintFactory(Providers.of(dao), null); assertNull(factory.getBlueprint(BLUEPRINT_NAME)); } @@ -228,17 +219,6 @@ public Blueprint createMultiInstanceBlueprint() throws Exception { return blueprint; } - @Test(expected=NoSuchStackException.class) - public void testCreateInvalidStack() throws Exception { - // setup mock to throw exception, to simulate invalid stack request - expect(stackFactory.createStack(new StackId())).andThrow(new StackAccessException("Invalid Stack")); - replay(stackFactory); - - BlueprintFactory factoryUnderTest = - new BlueprintFactory(stackFactory); - factoryUnderTest.createStack(new StackId()); - } - @Test(expected=IllegalArgumentException.class) public void testCreate_NoBlueprintName() throws Exception { Map props = BlueprintResourceProviderTest.getBlueprintTestProperties().iterator().next(); @@ -331,24 +311,4 @@ public void verifyStackDefinitionsAreDisjointShouldAllowDisjointStacks() { // no exception expected } - private class TestBlueprintFactory extends BlueprintFactory { - private Stack stack; - - public TestBlueprintFactory(Stack stack) { - this.stack = stack; - } - - @Override - protected Stack createStack(StackId stackId) throws NoSuchStackException { - return stack; - } - } - - - private void setPrivateField(Object o, String field, Object value) throws Exception { - Class c = o.getClass(); - Field f = c.getDeclaredField(field); - f.setAccessible(true); - f.set(o, value); - } } From d5347f6ae2f966a5d14e752d14069310bfd47338 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 6 Feb 2018 10:50:06 +0100 Subject: [PATCH 12/25] Remove unused dependencyConditionalServiceMap --- .../controller/internal/CompositeStack.java | 9 - .../server/controller/internal/Stack.java | 21 +- .../controller/internal/StackDefinition.java | 10 - .../topology/BlueprintValidatorImpl.java | 7 - .../internal/BaseBlueprintProcessorTest.java | 550 ------------------ 5 files changed, 1 insertion(+), 596 deletions(-) delete mode 100644 ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java index 75baf300bcf..b3316738476 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java @@ -246,15 +246,6 @@ public Collection getDependenciesForComponent(String component) .collect(toSet()); } - @Override - public String getConditionalServiceForDependency(DependencyInfo dependency) { - return stacks.stream() - .map(m -> m.getConditionalServiceForDependency(dependency)) - .filter(Objects::nonNull) - .findAny() - .orElse(null); - } - @Override public String getExternalComponentConfig(String component) { return stacks.stream() diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index 5f3186bdc54..15038f195e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -74,12 +74,6 @@ public class Stack implements StackDefinition { private Map> dependencies = new HashMap<>(); - /** - * Map of dependency to conditional service - */ - private Map dependencyConditionalServiceMap = - new HashMap<>(); - /** * Map of database component name to configuration property which indicates whether * the database in to be managed or if it is an external non-managed instance. @@ -170,10 +164,6 @@ public StackId getStackId() { return new StackId(getName(), getVersion()); } - Map getDependencyConditionalServiceMap() { - return dependencyConditionalServiceMap; - } - @Override public Set getStackIds() { return ImmutableSet.of(getStackId()); @@ -422,11 +412,6 @@ public Collection getDependenciesForComponent(String component) Collections.emptySet(); } - @Override - public String getConditionalServiceForDependency(DependencyInfo dependency) { - return dependencyConditionalServiceMap.get(dependency); - } - @Override public String getExternalComponentConfig(String component) { return dbDependencyInfo.get(component); @@ -610,11 +595,7 @@ private void parseExcludedConfigurations(ServiceInfo stackServiceResponse) { excludedConfigurationTypes.put(stackServiceResponse.getName(), stackServiceResponse.getExcludedConfigTypes()); } - /** - * Register conditional dependencies. - */ - //todo: This information should be specified in the stack definition. - void registerConditionalDependencies() { + private void registerConditionalDependencies() { dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java index 82aaa693ee2..201f554cf08 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java @@ -232,16 +232,6 @@ public interface StackDefinition { //todo: full dependency graph Collection getDependenciesForComponent(String component); - /** - * Get the service, if any, that a component dependency is conditional on. - * - * @param dependency dependency to get conditional service for - * - * @return conditional service for provided component or null if dependency - * is not conditional on a service - */ - String getConditionalServiceForDependency(DependencyInfo dependency); - /** * Get the custom "descriptor" that is used to decide whether component * is a managed or non-managed dependency. The descriptor is formatted as: diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index 1915398bd00..1e7f0ec0844 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -217,13 +217,6 @@ private Map> validateHostGroup(Blueprint blue for (DependencyInfo dependency : stack.getDependenciesForComponent(component.getName())) { LOGGER.debug("Processing dependency [{}] for component [{}]", dependency.getName(), component); - String conditionalService = stack.getConditionalServiceForDependency(dependency); - if (conditionalService != null && !blueprint.getServices().contains(conditionalService)) { - LOGGER.debug("Conditional service [{}] is missing from the blueprint, skipping dependency [{}]", - conditionalService, dependency.getName()); - continue; - } - // dependent components from the stack definitions are only added if related services are explicitly added to the blueprint! boolean isClientDependency = stack.getComponentInfo(dependency.getComponentName()).isClient(); if (isClientDependency && !blueprint.getServices().contains(dependency.getServiceName())) { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java deleted file mode 100644 index 341e889fdad..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java +++ /dev/null @@ -1,550 +0,0 @@ -package org.apache.ambari.server.controller.internal; - -import static org.junit.Assert.assertEquals; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.state.StackInfo; -import org.junit.Test; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -@SuppressWarnings("unchecked") -public class BaseBlueprintProcessorTest { - - //todo: Move these tests to the correct location. - //todo: BaseBluprintProcess no longer exists. - @Test - public void testStackRegisterConditionalDependencies() throws Exception { - StackInfo stackInfo = new StackInfo(); - - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(stackInfo) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 5, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoHCAT() throws Exception { - // test dependencies - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoYarnClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoTezClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoMapReduceClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoOozieClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - } - - /** - * Convenience class for easier setup/initialization of dependencies for unit - * testing. - */ - private static class TestDependencyInfo extends DependencyInfo { - TestDependencyInfo(String dependencyName) { - setName(dependencyName); - } - } -} \ No newline at end of file From 31a860fe4289ff455d048affe20f1e1f51e66c10 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 6 Feb 2018 14:51:35 +0100 Subject: [PATCH 13/25] Blueprint does not know about services --- .../StackAdvisorBlueprintProcessor.java | 6 +- .../server/controller/ControllerModule.java | 6 +- .../BlueprintConfigurationProcessor.java | 35 +++-- .../internal/BlueprintResourceProvider.java | 24 ++-- .../internal/ExportBlueprintRequest.java | 4 +- .../internal/ProvisionClusterRequest.java | 11 +- .../controller/internal/StackDefinition.java | 3 +- .../controller/internal/UnitUpdater.java | 2 +- .../ambari/server/topology/AmbariContext.java | 89 +++++++++--- .../ambari/server/topology/Blueprint.java | 85 ----------- .../server/topology/BlueprintFactory.java | 39 +---- .../ambari/server/topology/BlueprintImpl.java | 116 ++------------- .../topology/ClusterConfigurationRequest.java | 29 ++-- .../server/topology/ClusterTopology.java | 74 +++++++++- .../server/topology/ClusterTopologyImpl.java | 101 ++++++++++--- .../ambari/server/topology/Component.java | 9 +- .../GPLLicenseNotAcceptedException.java | 2 +- .../ambari/server/topology/HostGroup.java | 53 ------- .../ambari/server/topology/HostGroupImpl.java | 128 ++--------------- .../ambari/server/topology/HostRequest.java | 6 +- .../server/topology/LogicalRequest.java | 4 +- .../ambari/server/topology/Setting.java | 2 +- .../server/topology/TopologyManager.java | 8 +- .../topology/TopologyRequestFactoryImpl.java | 1 + .../tasks/PersistHostResourcesTask.java | 22 +-- .../validators/BasicBlueprintValidator.java | 29 ++++ .../{ => validators}/BlueprintValidator.java | 24 +--- .../ClusterConfigTypeValidator.java | 4 +- .../DependencyAndCardinalityValidator.java} | 103 ++++--------- .../validators/GplPropertiesValidator.java | 97 +++++++++++++ .../validators/HiveServiceValidator.java | 26 ++-- .../NameNodeHighAvailabilityValidator.java | 93 ++++++++++++ .../validators/RejectUnknownComponents.java | 45 ++++++ .../RequiredConfigPropertiesValidator.java | 44 +++--- .../validators/RequiredPasswordValidator.java | 2 +- .../validators/SecretReferenceValidator.java | 62 ++++++++ .../validators/StackConfigTypeValidator.java | 2 +- .../validators/TopologyValidatorFactory.java | 10 +- .../topology/validators/UnitValidator.java | 2 +- .../render/ClusterBlueprintRendererTest.java | 2 +- .../StackAdvisorBlueprintProcessorTest.java | 32 ++--- .../BlueprintConfigurationProcessorTest.java | 55 +++---- .../BlueprintResourceProviderTest.java | 49 ++----- .../internal/RequestResourceProviderTest.java | 5 +- .../controller/internal/UnitUpdaterTest.java | 2 +- .../server/topology/AmbariContextTest.java | 50 +++---- .../server/topology/BlueprintFactoryTest.java | 20 --- .../server/topology/BlueprintImplTest.java | 115 ++++----------- .../ClusterConfigurationRequestTest.java | 46 +++--- .../ClusterDeployWithStartOnlyTest.java | 60 ++++---- ...stallWithoutStartOnComponentLevelTest.java | 60 ++++---- .../ClusterInstallWithoutStartTest.java | 58 +++++--- .../topology/ClusterTopologyImplTest.java | 13 +- .../server/topology/LogicalRequestTest.java | 19 +-- .../server/topology/TopologyManagerTest.java | 48 ++++--- .../ClusterConfigTypeValidatorTest.java | 8 +- ...ependencyAndCardinalityValidatorTest.java} | 70 +++++---- .../GplPropertiesValidatorTest.java | 122 ++++++++++++++++ .../validators/HiveServiceValidatorTest.java | 136 ++++++++---------- .../RejectUnknownComponentsTest.java | 85 +++++++++++ ...RequiredConfigPropertiesValidatorTest.java | 6 +- .../RequiredPasswordValidatorTest.java | 18 ++- .../SecretReferenceValidatorTest.java | 59 ++++++++ .../StackConfigTypeValidatorTest.java | 2 +- .../validators/TopologyValidatorTests.java | 46 ++++++ .../validators/UnitValidatorTest.java | 2 +- 66 files changed, 1434 insertions(+), 1156 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java rename ambari-server/src/main/java/org/apache/ambari/server/topology/{ => validators}/BlueprintValidator.java (51%) rename ambari-server/src/main/java/org/apache/ambari/server/topology/{BlueprintValidatorImpl.java => validators/DependencyAndCardinalityValidator.java} (76%) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java rename ambari-server/src/test/java/org/apache/ambari/server/topology/{BlueprintValidatorImplTest.java => validators/DependencyAndCardinalityValidatorTest.java} (87%) create mode 100644 ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java create mode 100644 ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java rename ambari-server/src/test/java/org/apache/ambari/server/topology/{ => validators}/RequiredPasswordValidatorTest.java (94%) create mode 100644 ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java create mode 100644 ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java index 94ba563ed1f..b6f8d667e6e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java @@ -31,7 +31,6 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.AdvisedConfiguration; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.HostGroup; @@ -96,7 +95,7 @@ private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTop hgHostsMap); return StackAdvisorRequest.StackAdvisorRequestBuilder .forStack(stackId) - .forServices(clusterTopology.getBlueprint().getStack().getServices(stackId)) + .forServices(clusterTopology.getStack().getServices(stackId)) .forHosts(gatherHosts(clusterTopology)) .forHostsGroupBindings(gatherHostGroupBindings(clusterTopology)) .forHostComponents(gatherHostGroupComponents(clusterTopology)) @@ -176,12 +175,11 @@ private void addAdvisedConfigurationsToTopology(RecommendationResponse response, Map recommendedConfigurations = response.getRecommendations().getBlueprint().getConfigurations(); - Blueprint blueprint = topology.getBlueprint(); for (Map.Entry configEntry : recommendedConfigurations.entrySet()) { String configType = configEntry.getKey(); // add recommended config type only if related service is present in Blueprint - if (blueprint.isValidConfigType(configType)) { + if (topology.isValidConfigType(configType)) { BlueprintConfigurations blueprintConfig = filterBlueprintConfig(configType, configEntry.getValue(), userProvidedConfigurations, topology); topology.getAdvisedConfigurations().put(configType, new AdvisedConfiguration( diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index 5cfea7c2aca..1adc8474ab6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -169,14 +169,14 @@ import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.BlueprintValidatorImpl; import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; import org.apache.ambari.server.topology.PersistedStateImpl; import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.PasswordUtils; import org.apache.ambari.server.view.ViewInstanceHandlerList; import org.eclipse.jetty.server.SessionIdManager; @@ -543,7 +543,7 @@ private void installFactories() { bind(RegistryFactory.class).to(RegistryFactoryImpl.class); bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class); bind(SecurityHelper.class).toInstance(SecurityHelperImpl.getInstance()); - bind(BlueprintValidator.class).to(BlueprintValidatorImpl.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); bind(StackFactory.class).to(DefaultStackFactory.class); bind(BlueprintFactory.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index a167c098d07..3687ef90908 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -434,7 +434,7 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept setStackToolsAndFeatures(clusterConfig, configTypesUpdated); setRetryConfiguration(clusterConfig, configTypesUpdated); setupHDFSProxyUsers(clusterConfig, configTypesUpdated); - addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getBlueprint().getStack()); + addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getStack()); trimProperties(clusterConfig, clusterTopology); @@ -442,8 +442,7 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept } private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) { - Blueprint blueprint = clusterTopology.getBlueprint(); - StackDefinition stack = blueprint.getStack(); + StackDefinition stack = clusterTopology.getStack(); Map> configTypes = clusterConfig.getFullProperties(); for (String configType : configTypes.keySet()) { @@ -553,7 +552,7 @@ private void doFilterPriorToExport(Configuration configuration) { for (Map.Entry> configEntry : properties.entrySet()) { String type = configEntry.getKey(); try { - clusterTopology.getBlueprint().getStack().getServiceForConfigType(type); + clusterTopology.getStack().getServiceForConfigType(type); } catch (IllegalArgumentException illegalArgumentException) { LOG.error(new StringBuilder(String.format("Error encountered while trying to obtain the service name for config type [%s]. ", type)) .append("Further processing on this config type will be skipped. ") @@ -628,7 +627,7 @@ private void doRecommendConfigurations(Configuration configuration, Set */ private void doFilterStackDefaults(Map advisedConfigurations) { Blueprint blueprint = clusterTopology.getBlueprint(); - Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices()); + Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServices()); Map> stackDefaultProps = stackDefaults.getProperties(); for (Map.Entry adConfEntry : advisedConfigurations.entrySet()) { AdvisedConfiguration advisedConfiguration = adConfEntry.getValue(); @@ -1465,7 +1464,7 @@ public String updateForClusterCreate(String propertyName, topology.getHostAssignmentsForComponent(component).iterator().next(), properties); } else { //todo: extract all hard coded HA logic - Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component); + Cardinality cardinality = topology.getStack().getCardinality(component); // if no matching host groups are found for a component whose configuration // is handled by this updater, check the stack first to determine if // zero is a valid cardinality for this component. This is necessary @@ -1574,7 +1573,7 @@ public Collection getRequiredHostGroups(String propertyName, if (matchingGroupCount != 0) { return new HashSet<>(matchingGroups); } else { - Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component); + Cardinality cardinality = topology.getStack().getCardinality(component); // if no matching host groups are found for a component whose configuration // is handled by this updater, return an empty set if (! cardinality.isValidCount(0)) { @@ -2586,7 +2585,7 @@ public String updateForClusterCreate(String propertyName, } } - boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS"); + boolean isAtlasInCluster = topology.getServices().contains("ATLAS"); boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook")); // Append atlas hook if not already present. @@ -2617,7 +2616,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("ATLAS")) { + if (topology.getServices().contains("ATLAS")) { // if original value is not set or is the default "primary" set the cluster id if (origValue == null || origValue.trim().isEmpty() || origValue.equals("primary")) { //use cluster id because cluster name may change @@ -2652,7 +2651,7 @@ public String updateForClusterCreate(String propertyName, String origValue, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("ATLAS")) { + if (topology.getServices().contains("ATLAS")) { String host = topology.getHostAssignmentsForComponent("ATLAS_SERVER").iterator().next(); boolean tlsEnabled = Boolean.parseBoolean(properties.get("application-properties").get("atlas.enableTLS")); @@ -2710,7 +2709,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) { + if (topology.getServices().contains("AMBARI_METRICS")) { final String amsReporterClass = "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReporterClass; @@ -2741,7 +2740,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) { + if (topology.getServices().contains("AMBARI_METRICS")) { final String amsReportesClass = "org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReportesClass; @@ -2833,7 +2832,7 @@ private Collection setupHDFSProxyUsers(Configuration configuration, Set< // AMBARI-5206 final Map userProps = new HashMap<>(); - Collection services = clusterTopology.getBlueprint().getServices(); + Collection services = clusterTopology.getServices(); if (services.contains("HDFS")) { // only add user properties to the map for // services actually included in the blueprint definition @@ -2887,13 +2886,13 @@ private Collection setupHDFSProxyUsers(Configuration configuration, Set< * @param stack */ private void addExcludedConfigProperties(Configuration configuration, Set configTypesUpdated, StackDefinition stack) { - Collection blueprintServices = clusterTopology.getBlueprint().getServices(); + Collection blueprintServices = clusterTopology.getServices(); - LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices); + LOG.info("Handling excluded properties for blueprint services: {}", blueprintServices); for (String blueprintService : blueprintServices) { - LOG.debug("Handling excluded properties for blueprint service: {}", blueprintService); + LOG.info("Handling excluded properties for blueprint service: {}", blueprintService); Set excludedConfigTypes = stack.getExcludedConfigurationTypes(blueprintService); if (excludedConfigTypes.isEmpty()) { @@ -3110,7 +3109,7 @@ private static class StackPropertyTypeFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); final String serviceName = stack.getServiceForConfigType(configType); return !(stack.isPasswordProperty(serviceName, configType, propertyName) || stack.isKerberosPrincipalNameProperty(serviceName, configType, propertyName)); @@ -3207,7 +3206,7 @@ private static abstract class DependencyFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); Configuration configuration = topology.getConfiguration(); final String serviceName = stack.getServiceForConfigType(configType); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 760d90f89c0..8b45c4110d6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -59,12 +59,10 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; -import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.JsonUtils; import org.apache.ambari.server.utils.SecretReference; import org.slf4j.Logger; @@ -545,19 +543,8 @@ public Void invoke() throws AmbariException { blueprint.getName()); } - try { - validator.validateRequiredProperties(blueprint); - } catch (InvalidTopologyException | GPLLicenseNotAcceptedException e) { - throw new IllegalArgumentException("Blueprint configuration validation failed: " + e.getMessage(), e); - } - - String validateTopology = requestInfoProps.get(VALIDATE_TOPOLOGY_PROPERTY_ID); - if (validateTopology == null || ! validateTopology.equalsIgnoreCase("false")) { - try { - validator.validateTopology(blueprint); - } catch (InvalidTopologyException e) { - throw new IllegalArgumentException(e.getMessage()); - } + if (shouldValidate(requestInfoProps)) { + validator.validate(blueprint); } LOG.info("Creating Blueprint, name=" + blueprint.getName()); @@ -574,6 +561,11 @@ public Void invoke() throws AmbariException { }; } + private static boolean shouldValidate(Map requestInfoProps) { + String validateTopology = requestInfoProps.get("validate_topology"); + return validateTopology == null || Boolean.parseBoolean(validateTopology); + } + /** * The structure of blueprints is evolving where multiple resource * structures are to be supported. This class abstracts the population diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java index ac30db97040..151d9a7c3ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java @@ -135,11 +135,11 @@ private void createBlueprint(Collection exportedHostGroups, S componentList.add(new Component(component)); } - hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList, + hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), componentList, exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality()))); } ImmutableSet stackIds = ImmutableSet.of(stack.getStackId()); - blueprint = new BlueprintImpl(bpName, hostGroups, stack, stackIds, Collections.emptySet(), configuration, null, null); + blueprint = new BlueprintImpl(bpName, hostGroups, stackIds, Collections.emptySet(), configuration, null, null); } private void createHostGroupInfo(Collection exportedHostGroups) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index 99de97c9458..1af9248d021 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -49,8 +49,6 @@ import com.google.common.base.Enums; import com.google.common.base.Optional; import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; /** * Request for provisioning a cluster. @@ -164,7 +162,6 @@ public class ProvisionClusterRequest extends BaseClusterRequest { private final Collection mpackInstances; private final Set stackIds; - private final StackDefinition stack; private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class); @@ -215,9 +212,7 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura setProvisionAction(parseProvisionAction(properties)); mpackInstances = BlueprintFactory.createMpackInstances(properties); - Set stackIdsInRequest = mpackInstances.stream().map(MpackInstance::getStackId).collect(toSet()); // FIXME persist these - stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), stackIdsInRequest)); - stack = blueprintFactory.composeStacks(stackIds); + stackIds = mpackInstances.stream().map(MpackInstance::getStackId).collect(toSet()); // FIXME persist these try { this.quickLinksProfileJson = processQuickLinksProfile(properties); @@ -527,10 +522,6 @@ public Set getStackIds() { return stackIds; } - public StackDefinition getStack() { - return stack; - } - public Collection getMpackInstances() { return mpackInstances; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java index 201f554cf08..29041e681db 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java @@ -30,6 +30,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.validators.DependencyAndCardinalityValidator; /** * Encapsulates stack information. @@ -240,7 +241,7 @@ public interface StackDefinition { * * @param component component to get dependency information for * @return the descriptor of form "config_type/property_name" - * @see org.apache.ambari.server.topology.BlueprintValidatorImpl#isDependencyManaged + * @see DependencyAndCardinalityValidator#isDependencyManaged */ String getExternalComponentConfig(String component); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java index bb02b44febc..0e57599fd87 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java @@ -49,7 +49,7 @@ public String updateForClusterCreate(String propertyName, String origValue, Map> properties, ClusterTopology topology) { - PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName); + PropertyUnit stackUnit = PropertyUnit.of(topology.getStack(), serviceName, configType, propertyName); PropertyValue value = PropertyValue.of(propertyName, origValue); if (value.hasUnit(stackUnit)) { return value.toString(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index f4058a1e79d..8143c9fcb48 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toSet; import java.util.Collection; @@ -34,6 +35,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; +import java.util.function.Function; +import java.util.stream.Stream; import javax.annotation.Nullable; import javax.inject.Inject; @@ -43,6 +46,7 @@ import org.apache.ambari.server.DuplicateResourceException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; +import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.actionmanager.HostRoleStatus; @@ -68,6 +72,7 @@ import org.apache.ambari.server.controller.internal.ServiceGroupDependencyResourceProvider; import org.apache.ambari.server.controller.internal.ServiceGroupResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; +import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; @@ -80,6 +85,7 @@ import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.security.authorization.AuthorizationException; +import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -92,10 +98,12 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.utils.RetryHelper; +import org.apache.commons.lang3.tuple.Pair; import org.apache.directory.api.util.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; @@ -121,6 +129,9 @@ public enum TaskType {INSTALL, START} @Inject ConfigFactory configFactory; + @Inject + StackFactory stackFactory; + @Inject RepositoryVersionDAO repositoryVersionDAO; @@ -344,7 +355,7 @@ public Object call() throws Exception { public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { Set serviceGroups = Sets.newHashSet(DEFAULT_SERVICE_GROUP_NAME); - Collection services = topology.getBlueprint().getServices(); + Collection services = topology.getServices(); try { Cluster cluster = getController().getClusters().getCluster(clusterName); @@ -361,15 +372,15 @@ public void createAmbariServiceAndComponentResources(ClusterTopology topology, S Set serviceRequests = new HashSet<>(); Set componentRequests = new HashSet<>(); for (String service : services) { - String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service); - StackId stackId = Iterables.getOnlyElement(topology.getBlueprint().getStackIdsForService(service)); // FIXME temporarily assume each service is defined in only one mpack + String credentialStoreEnabled = topology.getBlueprint().getSetting().getCredentialStoreEnabled(service); + StackId stackId = Iterables.getOnlyElement(topology.getStackIdsForService(service)); // FIXME temporarily assume each service is defined in only one mpack Long repositoryVersionId = repoVersionByStack.get(stackId); serviceRequests.add(new ServiceRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, service, repositoryVersionId, null, credentialStoreEnabled, stackId )); - for (String component : topology.getBlueprint().getComponentNames(service)) { - String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component); + for (String component : topology.getComponentNames(service)) { + String recoveryEnabled = topology.getBlueprint().getSetting().getRecoveryEnabled(service, component); componentRequests.add(new ServiceComponentRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, null, recoveryEnabled)); } } @@ -406,7 +417,7 @@ public void createAmbariServiceAndComponentResources(ClusterTopology topology, S } } - public void createAmbariHostResources(long clusterId, String hostName, Map> components) { + public void createAmbariHostResources(long clusterId, String hostName, Collection components) { Host host; try { host = getController().getClusters().getHost(hostName); @@ -440,17 +451,15 @@ public void createAmbariHostResources(long clusterId, String hostName, Map requests = new HashSet<>(); - for (Map.Entry> entry : components.entrySet()) { - String service = entry.getKey(); - for (String component : entry.getValue()) { - //todo: handle this in a generic manner. These checks are all over the code - try { - if (cluster.getService(service) != null && !component.equals(RootComponent.AMBARI_SERVER.name())) { - requests.add(new ServiceComponentHostRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, hostName, null)); - } - } catch(AmbariException se) { - LOG.warn("Service already deleted from cluster: {}", service); + for (Component component : components) { + String serviceName = component.getServiceInstance(); + String componentName = component.getName(); + try { + if (cluster.getService(serviceName) != null && !componentName.equals(RootComponent.AMBARI_SERVER.name())) { + requests.add(new ServiceComponentHostRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, serviceName, componentName, hostName, null)); } + } catch (AmbariException e) { + LOG.warn("Service already deleted from cluster: {}", serviceName); } } try { @@ -760,7 +769,7 @@ private boolean addHostToExistingConfigGroups(String hostName, ClusterTopology t */ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String groupName) throws AmbariException { Map> groupConfigs = new HashMap<>(); - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); // get the host-group config with cluster creation template overrides Configuration topologyHostGroupConfig = topology. @@ -774,7 +783,7 @@ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String for (Map.Entry> entry : userProvidedGroupProperties.entrySet()) { String type = entry.getKey(); String service = stack.getServicesForConfigType(type) - .filter(each -> topology.getBlueprint().getServices().contains(each)) + .filter(each -> topology.getServices().contains(each)) .findFirst() // TODO check if this is required at all (might be handled by the "orphan" removal) // TODO move this validation earlier @@ -834,6 +843,50 @@ public boolean apply(@Nullable String groupHost) { } } + public StackDefinition composeStacks(Set stackIds) { + Set stacks = stackIds.stream() + .map(this::createStack) + .collect(toSet()); + StackDefinition composite = StackDefinition.of(stacks); + + // temporary check + verifyStackDefinitionsAreDisjoint(composite.getServices().stream(), "Service", composite::getStacksForService); + verifyStackDefinitionsAreDisjoint(composite.getComponents().stream(), "Component", composite::getStacksForComponent); + + return composite; + } + + /** + * Verify that each item in items is defined by only one stack. + * + * @param items the items to check + * @param type string description of the type of items (eg. "Service", or "Component") + * @param lookup a function to find the set of stacks that an item belongs to + * @throws IllegalArgumentException if some items are defined in multiple stacks + */ + static void verifyStackDefinitionsAreDisjoint(Stream items, String type, Function> lookup) { + Set>> definedInMultipleStacks = items + .map(s -> Pair.of(s, lookup.apply(s))) + .filter(p -> p.getRight().size() > 1) + .collect(toCollection(TreeSet::new)); + + if (!definedInMultipleStacks.isEmpty()) { + String msg = definedInMultipleStacks.stream() + .map(p -> String.format("%s %s is defined in multiple stacks: %s", type, p.getLeft(), Joiner.on(", ").join(p.getRight()))) + .collect(joining("\n")); + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + + protected Stack createStack(StackId stackId) { + try { + return stackFactory.createStack(stackId); + } catch (StackAccessException e) { + throw new NoSuchStackException(stackId); + } + } + /** * Get a config group name based on a bp and host group. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 04e004e5c55..727e42ecd8a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -23,7 +23,6 @@ import java.util.Map; import java.util.Set; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.state.StackId; @@ -73,80 +72,11 @@ public interface Blueprint { */ Setting getSetting(); - /** - * Get all of the services represented in the blueprint. - * - * @return collection of all represented service names - */ - Collection getServices(); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of components for the service. Will not return null. - */ - Collection getComponents(String service); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of component names for the service. Will not return null. - */ - @Deprecated - Collection getComponentNames(String service); - - /** - * Get whether a component is enabled for auto start. - * - * @param serviceName - Service name. - * @param componentName - Component name. - * - * @return null if value is not specified; true or false if specified. - */ - String getRecoveryEnabled(String serviceName, String componentName); - - /** - * Get whether a service is enabled for credential store use. - * - * @param serviceName - Service name. - * - * @return null if value is not specified; true or false if specified. - */ - String getCredentialStoreEnabled(String serviceName); - - /** - * Check if auto skip failure is enabled. - * @return true if enabled, otherwise false. - */ - boolean shouldSkipFailure(); - - /** - * Get the stack associated with the blueprint. - * For mpack-based installation this is a composite stack - * that provides a single unified view of all underlying mpacks, - * but does not have any identifier. - * - * @return associated stack - */ - StackDefinition getStack(); - /** * @return the set of stack (mpack) IDs associated with the blueprint */ Set getStackIds(); - /** - * Look up the stacks that define service. - * To be used only after checking that services map to - * @param service the name of the service as defined in the stack (mpack), eg. ZOOKEEPER - * @return the ID of the stack that defines the given service - */ - Set getStackIdsForService(String service); - /** * Get the mpacks associated with the blueprint. * @@ -154,16 +84,6 @@ public interface Blueprint { */ Collection getMpacks(); - /** - * Get the host groups which contain components for the specified service. - * - * @param service service name - * - * @return collection of host groups containing components for the specified service; - * will not return null - */ - Collection getHostGroupsForService(String service); - /** * Get the host groups which contain the give component. * @@ -175,11 +95,6 @@ public interface Blueprint { SecurityConfiguration getSecurity(); - /** - * A config type is valid if there are services related to except cluster-env and global. - */ - boolean isValidConfigType(String configType); - /** * Obtain the blueprint as an entity. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 9ae3a3c8cc4..aee21e1edd5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -40,7 +40,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -51,7 +50,6 @@ import javax.inject.Provider; import org.apache.ambari.server.ObjectNotFoundException; -import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; @@ -91,8 +89,7 @@ public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException Set stackIds = entity.getMpackInstances().stream() .map(m -> new StackId(m.getMpackName(), m.getMpackVersion())) .collect(toSet()); - StackDefinition stack = composeStacks(stackIds); - return new BlueprintImpl(entity, stack, stackIds); + return new BlueprintImpl(entity, stackIds); } return null; } @@ -121,13 +118,12 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu Set stackIds = mpackInstances.stream() .map(MpackInstance::getStackId) .collect(toSet()); - StackDefinition stack = composeStacks(stackIds); - Collection hostGroups = processHostGroups(name, stack, properties); + Collection hostGroups = processHostGroups(properties); Configuration configuration = configFactory.getConfiguration((Collection>) properties.get(CONFIGURATION_PROPERTY_ID)); Setting setting = SettingFactory.getSetting((Collection>) properties.get(SETTING_PROPERTY_ID)); - return new BlueprintImpl(name, hostGroups, stack, stackIds, mpackInstances, configuration, securityConfiguration, setting); + return new BlueprintImpl(name, hostGroups, stackIds, mpackInstances, configuration, securityConfiguration, setting); } public static Collection createMpackInstances(Map properties) throws NoSuchStackException { @@ -198,7 +194,7 @@ private Stack createStack(StackId stackId) { //todo: Move logic to HostGroupImpl @SuppressWarnings("unchecked") - private Collection processHostGroups(String bpName, StackDefinition stack, Map properties) { + private Collection processHostGroups(Map properties) { Set> hostGroupProps = (Set>) properties.get(HOST_GROUP_PROPERTY_ID); @@ -219,23 +215,22 @@ private Collection processHostGroups(String bpName, StackDefinition s Collection> configProps = (Collection>) hostGroupProperties.get(CONFIGURATION_PROPERTY_ID); - Collection components = processHostGroupComponents(stack, hostGroupName, componentProps); + Collection components = processHostGroupComponents(hostGroupName, componentProps); Configuration configuration = configFactory.getConfiguration(configProps); String cardinality = String.valueOf(hostGroupProperties.get(HOST_GROUP_CARDINALITY_PROPERTY_ID)); - HostGroup group = new HostGroupImpl(hostGroupName, bpName, stack, components, configuration, cardinality); + HostGroup group = new HostGroupImpl(hostGroupName, components, configuration, cardinality); hostGroups.add(group); } return hostGroups; } - private Collection processHostGroupComponents(StackDefinition stack, String groupName, Set> componentProps) { + private Collection processHostGroupComponents(String groupName, Set> componentProps) { if (componentProps == null || componentProps.isEmpty()) { throw new IllegalArgumentException("Host group '" + groupName + "' must contain at least one component"); } - Collection stackComponentNames = getAllStackComponents(stack); Collection components = new ArrayList<>(); for (Map componentProperties : componentProps) { @@ -245,11 +240,6 @@ private Collection processHostGroupComponents(StackDefinition stack, "' contains a component with no 'name' property"); } - if (! stackComponentNames.contains(componentName)) { - throw new IllegalArgumentException("The component '" + componentName + "' in host group '" + - groupName + "' is not valid for the specified stack"); - } - String mpackInstance = componentProperties.get(COMPONENT_MPACK_INSTANCE_PROPERTY); String serviceInstance = componentProperties.get(COMPONENT_SERVICE_INSTANCE_PROPERTY); //TODO, might want to add some validation here, to only accept value enum types, rwn @@ -261,19 +251,4 @@ private Collection processHostGroupComponents(StackDefinition stack, return components; } - /** - * Obtain all component names for the specified stack. - * - * @return collection of component names for the specified stack - * @throws IllegalArgumentException if the specified stack doesn't exist - */ - private Collection getAllStackComponents(StackDefinition stack) { - Collection allComponents = new HashSet<>(stack.getComponents()); - - // currently ambari server is not a recognized component - allComponents.add(RootComponent.AMBARI_SERVER.name()); - - return allComponents; - } - } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 7e13d0d3099..fc636eb44f8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -19,8 +19,6 @@ package org.apache.ambari.server.topology; -import static java.util.stream.Collectors.toList; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,7 +30,6 @@ import java.util.Set; import java.util.function.Supplier; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintConfiguration; import org.apache.ambari.server.orm.entities.BlueprintEntity; @@ -43,7 +40,6 @@ import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.stack.NoSuchStackException; -import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.JsonUtils; @@ -61,37 +57,33 @@ public class BlueprintImpl implements Blueprint { private final String name; private final Map hostGroups; private final Collection mpacks; - private final StackDefinition stack; private final Set stackIds; private final Configuration configuration; private final SecurityConfiguration security; private final Setting setting; private final List repoSettings; - public BlueprintImpl(BlueprintEntity entity, StackDefinition stack, Set stackIds) throws NoSuchStackException { + public BlueprintImpl(BlueprintEntity entity, Set stackIds) throws NoSuchStackException { name = entity.getBlueprintName(); security = entity.getSecurityType() != null ? new SecurityConfiguration(entity.getSecurityType(), entity.getSecurityDescriptorReference(), null) : SecurityConfiguration.NONE; mpacks = parseMpacks(entity); - this.stack = stack; this.stackIds = stackIds; // create config first because it is set as a parent on all host-group configs configuration = processConfiguration(entity.getConfigurations()); hostGroups = parseBlueprintHostGroups(entity); - // TODO: how to handle multiple stacks correctly? - configuration.setParentConfiguration(stack.getConfiguration(getServices())); + // configuration.setParentConfiguration(stack.getConfiguration(getServices())); // FIXME services not available at this time, need to set parent config somewhere else setting = new Setting(parseSetting(entity.getSettings())); repoSettings = processRepoSettings(); } - public BlueprintImpl(String name, Collection groups, StackDefinition stack, Set stackIds, Collection mpacks, + public BlueprintImpl(String name, Collection groups, Set stackIds, Collection mpacks, Configuration configuration, SecurityConfiguration security, Setting setting) { this.name = name; this.mpacks = mpacks; - this.stack = stack; this.stackIds = stackIds; this.security = security != null ? security : SecurityConfiguration.NONE; @@ -100,12 +92,12 @@ public BlueprintImpl(String name, Collection groups, StackDefinition for (HostGroup hostGroup : groups) { hostGroups.put(hostGroup.getName(), hostGroup); } - // TODO: handle configuration from multiple stacks properly - // if the parent isn't set, the stack configuration is set as the parent this.configuration = configuration; - if (configuration.getParentConfiguration() == null) { - configuration.setParentConfiguration(stack.getConfiguration(getServices())); - } + // if the parent isn't set, the stack configuration is set as the parent + // parent is set to non-null, but empty config when exporting blueprint to prevent exporting stack default config +// if (configuration.getParentConfiguration() == null) { +// configuration.setParentConfiguration(stack.getConfiguration(getServices())); // FIXME services not available at this time +// } this.setting = setting != null ? setting : new Setting(ImmutableMap.of()); repoSettings = processRepoSettings(); } @@ -118,11 +110,6 @@ public Set getStackIds() { return stackIds; } - @Override - public Set getStackIdsForService(String service) { - return stack.getStacksForService(service); - } - public SecurityConfiguration getSecurity() { return security; } @@ -149,66 +136,10 @@ public Setting getSetting() { return setting; } - /** - * Get all services represented in blueprint. - * - * @return collections of all services provided by topology - */ - @Override - public Collection getServices() { - Collection services = new HashSet<>(); - for (HostGroup group : getHostGroups().values()) { - services.addAll(group.getServices()); - } - return services; - } - - @Override - public Collection getComponents(String service) { - Collection components = new HashSet<>(); - for (HostGroup group : getHostGroupsForService(service)) { - components.addAll(group.getComponents(service)); - } - return components; - } - - @Override - @Deprecated - public Collection getComponentNames(String service) { - return getComponents(service).stream().map(Component::getName).collect(toList()); - } - - @Override - public String getRecoveryEnabled(String serviceName, String componentName) { - return setting.getRecoveryEnabled(serviceName, componentName); - } - - @Override - public String getCredentialStoreEnabled(String serviceName) { - return setting.getCredentialStoreEnabled(serviceName); - } - - @Override - public boolean shouldSkipFailure() { - return setting.shouldSkipFailure(); - } - - @Override public Collection getMpacks() { return mpacks; } - public StackDefinition getStack() { - return stack; - } - - /** - * Get host groups which contain a component. - * - * @param component component name - * - * @return collection of host groups which contain the specified component - */ @Override public Collection getHostGroupsForComponent(String component) { Collection resultGroups = new HashSet<>(); @@ -220,24 +151,6 @@ public Collection getHostGroupsForComponent(String component) { return resultGroups; } - /** - * Get host groups which contain a component for the given service. - * - * @param service service name - * - * @return collection of host groups which contain a component of the specified service - */ - @Override - public Collection getHostGroupsForService(String service) { - Collection resultGroups = new HashSet<>(); - for (HostGroup group : hostGroups.values() ) { - if (group.getServices().contains(service)) { - resultGroups.add(group); - } - } - return resultGroups; - } - public BlueprintEntity toEntity() { BlueprintEntity entity = new BlueprintEntity(); entity.setBlueprintName(name); @@ -292,7 +205,7 @@ private Collection parseMpacks(BlueprintEntity blueprintEntity) t private Map parseBlueprintHostGroups(BlueprintEntity entity) { Map hostGroups = new HashMap<>(); for (HostGroupEntity hostGroupEntity : entity.getHostGroups()) { - HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), getStack()); + HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity); // set the bp configuration as the host group config parent hostGroup.getConfiguration().setParentConfiguration(configuration); hostGroups.put(hostGroupEntity.getName(), hostGroup); @@ -518,17 +431,6 @@ private void createBlueprintSettingEntities(BlueprintEntity blueprintEntity) { } } - /** - * A config type is valid if there are services related to except cluster-env and global. - */ - public boolean isValidConfigType(String configType) { - if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) { - return true; - } - String service = getStack().getServiceForConfigType(configType); - return getServices().contains(service); - } - /** * Parse stack repo info stored in the blueprint_settings table */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index ee626bce84a..05fe8390bec 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -75,8 +75,7 @@ public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor) { this.ambariContext = ambariContext; this.clusterTopology = clusterTopology; - Blueprint blueprint = clusterTopology.getBlueprint(); - this.stack = blueprint.getStack(); + this.stack = clusterTopology.getStack(); // set initial configuration (not topology resolved) this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology); this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor; @@ -90,11 +89,9 @@ public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology * Remove config-types from the given configuration if there is no any services related to them (except cluster-env and global). */ private void removeOrphanConfigTypes(Configuration configuration) { - Blueprint blueprint = clusterTopology.getBlueprint(); - Collection configTypes = configuration.getAllConfigTypes(); for (String configType : configTypes) { - if (!blueprint.isValidConfigType(configType)) { + if (!clusterTopology.isValidConfigType(configType)) { configuration.removeConfigType(configType); LOG.info("Removing config type '{}' as related service is not present in either Blueprint or cluster creation template.", configType); } @@ -164,7 +161,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> stackDefaultProps = stackDefaults.getProperties(); // add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of @@ -176,7 +173,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map(blueprint.getServices())); + new HashSet<>(clusterTopology.getServices())); // apply Kerberos specific configurations Map> updatedConfigs = AmbariContext.getController().getKerberosHelper() @@ -198,7 +195,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map propertyMap = updatedConfigs.get(configType); Map clusterConfigProperties = existingConfigurations.get(configType); Map stackDefaultConfigProperties = stackDefaultProps.get(configType); @@ -233,15 +230,12 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> createServiceComponentMap(Blueprint blueprint) { Map> serviceComponents = new HashMap<>(); - Collection services = blueprint.getServices(); + Collection services = clusterTopology.getServices(); if(services != null) { for (String service : services) { - Collection components = blueprint.getComponentNames(service); - serviceComponents.put(service, - (components == null) - ? Collections.emptySet() - : new HashSet<>(blueprint.getComponentNames(service))); + Collection components = clusterTopology.getComponentNames(service); + serviceComponents.put(service, components == null ? Collections.emptySet() : new HashSet<>(components)); } } @@ -280,8 +274,8 @@ private boolean propertyHasCustomValue(Map clusterConfigProperti private Map createComponentHostMap(Blueprint blueprint) { Map componentHostsMap = new HashMap<>(); - for (String service : blueprint.getServices()) { - Collection components = blueprint.getComponentNames(service); + for (String service : clusterTopology.getServices()) { + Collection components = clusterTopology.getComponentNames(service); for (String component : components) { Collection componentHost = clusterTopology.getHostAssignmentsForComponent(component); // retrieve corresponding clusterInfoKey for component using StageUtils @@ -352,10 +346,9 @@ public void setConfigurationsOnCluster(ClusterTopology clusterTopology, String t //todo: also handle setting of host group scoped configuration which is updated by config processor List configurationRequests = new LinkedList<>(); - Blueprint blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); - for (String service : blueprint.getServices()) { + for (String service : clusterTopology.getServices()) { //todo: remove intermediate request type // one bp config request per service BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index dc89dd29fb8..a9ba9a975dd 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -21,9 +21,11 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.StackId; /** @@ -49,12 +51,22 @@ public interface ClusterTopology { /** * Get the blueprint associated with the cluster. * - * @return assocaited blueprint + * @return associated blueprint */ Blueprint getBlueprint(); /** - * @return the set of stack (mpack) IDs associated with the blueprint + * Get the stack associated with the blueprint. + * For mpack-based installation this is a composite stack + * that provides a single unified view of all underlying mpacks, + * but does not have any identifier. + * + * @return associated stack + */ + StackDefinition getStack(); + + /** + * @return the set of stack (mpack) IDs associated with the cluster */ Set getStackIds(); @@ -103,6 +115,58 @@ public interface ClusterTopology { */ Collection getHostAssignmentsForComponent(String component); + /** + * Get all of the services represented in the blueprint. + * + * @return collection of all represented service names + */ + Collection getServices(); + + /** + * Get all of the components represented in the blueprint. + * + * @return collection of all represented components + */ + Stream getComponents(); + + /** + * Get the names of all of the components represented in the blueprint. + * + * @return collection of all represented component names + */ + Stream getComponentNames(); + + /** + * Look up the stacks that define service. + * To be used only after checking that services map to + * @param service the name of the service as defined in the stack (mpack), eg. ZOOKEEPER + * @return the ID of the stack that defines the given service + */ + Set getStackIdsForService(String service); + + /** + * Get the components that are included in the blueprint for the specified service. + * + * @param service service name + * + * @return collection of components for the service. Will not return null. + */ + Collection getComponents(String service); + + /** + * Get the components that are included in the blueprint for the specified service. + * + * @param service service name + * + * @return collection of component names for the service. Will not return null. + */ + Collection getComponentNames(String service); + + /** + * A config type is valid if there are services related to except cluster-env and global. + */ + boolean isValidConfigType(String configType); + /** * Update the existing topology based on the provided topology request. * @@ -187,4 +251,10 @@ public interface ClusterTopology { String getDefaultPassword(); + /** + * Determine if the host group contains a master component. + * + * @return true if the host group contains a master component; false otherwise + */ + boolean containsMasterComponent(HostGroup hostGroup); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index f3f0fdfee63..79bb417dbc6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -19,6 +19,8 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; @@ -28,57 +30,62 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableSet; + /** * Represents a cluster topology. * Topology includes the the associated blueprint, cluster configuration and hostgroup -> host mapping. */ public class ClusterTopologyImpl implements ClusterTopology { - private Long clusterId; + private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class); - //todo: currently topology is only associated with a single bp - //todo: this will need to change to allow usage of multiple bp's for the same cluster - //todo: for example: provision using bp1 and scale using bp2 - private Blueprint blueprint; - private Configuration configuration; + private final Set stackIds; + private final StackDefinition stack; + private Long clusterId; + private final Blueprint blueprint; + private final Configuration configuration; private ConfigRecommendationStrategy configRecommendationStrategy; private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START; - private Map advisedConfigurations = new HashMap<>(); + private final Map advisedConfigurations = new HashMap<>(); private final Map hostGroupInfoMap = new HashMap<>(); private final AmbariContext ambariContext; private final String defaultPassword; - private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class); - - - //todo: will need to convert all usages of hostgroup name to use fully qualified name (BP/HG) - //todo: for now, restrict scaling to the same BP public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topologyRequest) throws InvalidTopologyException { + this.ambariContext = ambariContext; this.clusterId = topologyRequest.getClusterId(); // provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here this.blueprint = topologyRequest.getBlueprint(); this.configuration = topologyRequest.getConfiguration(); + if (topologyRequest instanceof ProvisionClusterRequest) { - this.defaultPassword = ((ProvisionClusterRequest) topologyRequest).getDefaultPassword(); + ProvisionClusterRequest provisionRequest = (ProvisionClusterRequest) topologyRequest; + defaultPassword = provisionRequest.getDefaultPassword(); + stackIds = provisionRequest.getStackIds(); } else { - this.defaultPassword = null; + defaultPassword = null; + stackIds = ImmutableSet.of(); } + stack = ambariContext.composeStacks(stackIds); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); // todo extract validation to specialized service validateTopology(); - this.ambariContext = ambariContext; } @Override @@ -103,7 +110,12 @@ public Blueprint getBlueprint() { @Override public Set getStackIds() { - return blueprint.getStackIds(); + return stackIds; + } + + @Override + public StackDefinition getStack() { + return stack; } @Override @@ -184,6 +196,63 @@ public Collection getHostAssignmentsForComponent(String component) { return hosts; } + @Override + public Collection getServices() { + return getComponentNames() + .map(stack::getServiceForComponent) + .collect(toSet()); + } + + @Override + public Stream getComponentNames() { + return getComponents() + .map(Component::getName); + } + + @Override + public Stream getComponents() { + return getBlueprint().getHostGroups().values().stream() + .flatMap(hostGroup -> hostGroup.getComponents().stream()); + } + + @Override + public Set getStackIdsForService(String service) { + return stack.getStacksForService(service); + } + + @Override + public Collection getComponents(String service) { + return getComponents() + .filter(c -> stack.getComponents(service).contains(c.getName())) + .collect(toSet()); + } + + @Override + public Collection getComponentNames(String service) { + return getComponents(service).stream().map(Component::getName).collect(toList()); + } + + @Override + public boolean containsMasterComponent(HostGroup hostGroup) { + return hostGroup.getComponentNames().stream() + .map(stack::isMasterComponent) + .findAny() + .orElse(false); + } + + @Override + public boolean isValidConfigType(String configType) { + if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) { + return true; + } + try { + String service = getStack().getServiceForConfigType(configType); + return getServices().contains(service); + } catch (IllegalArgumentException e) { + return false; + } + } + @Override public boolean isNameNodeHAEnabled() { return isNameNodeHAEnabled(configuration.getFullProperties()); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java index a62d341d3cc..3cc9c9931e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java @@ -28,11 +28,8 @@ public class Component { private final String name; - @Nullable private final String mpackInstance; - @Nullable private final String serviceInstance; - private final ProvisionAction provisionAction; @Deprecated @@ -40,7 +37,7 @@ public Component(String name) { this(name, null, null, null); } - public Component(String name, String mpackInstance, String serviceInstance, ProvisionAction provisionAction) { + public Component(String name, @Nullable String mpackInstance, @Nullable String serviceInstance, ProvisionAction provisionAction) { this.name = name; this.mpackInstance = mpackInstance; this.serviceInstance = serviceInstance; @@ -57,14 +54,14 @@ public String getName() { } /** - * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambigous) + * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambiguous) */ public String getMpackInstance() { return mpackInstance; } /** - * @return the service instance this component belongs to. Can be {@null} if component does not belong to a service + * @return the service instance this component belongs to. Can be {@code null} if component does not belong to a service * instance (there is a single service of the component's service type) */ public String getServiceInstance() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java index b444d01345f..fcc052e559e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java @@ -21,7 +21,7 @@ /** * Indicates an not permitted LZO usage. */ -public class GPLLicenseNotAcceptedException extends Exception { +public class GPLLicenseNotAcceptedException extends InvalidTopologyException { public GPLLicenseNotAcceptedException(String s) { super(s); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java index 24608059718..9ece5a26e75 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -21,7 +21,6 @@ import java.util.Collection; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.StackDefinition; /** * Host Group representation. @@ -35,13 +34,6 @@ public interface HostGroup { */ String getName(); - /** - * Get the name of the associated blueprint - * - * @return associated blueprint name - */ - String getBlueprintName(); - /** * Get all of the host group components. * @@ -49,7 +41,6 @@ public interface HostGroup { */ Collection getComponents(); - /** * Get all of the host group component names * @@ -71,48 +62,11 @@ public interface HostGroup { @Deprecated Collection getComponentNames(ProvisionAction provisionAction); - /** - * Get the names components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of component names - */ - @Deprecated - Collection getComponentNames(String service); - - /** - * Get the host group components which belong to the specified service. - * - * @param service service instance name or service name. First, services looked up - * by instance name. If no appropriate service instance is found, services are looked - * up by type - * - * @return collection of component names for the specified service; will not return null - */ - Collection getComponents(String service); - - - /** * Add a component to the host group */ boolean addComponent(Component component); - /** - * Determine if the host group contains a master component. - * - * @return true if the host group contains a master component; false otherwise - */ - boolean containsMasterComponent(); - - /** - * Get all of the services associated with the host group components. - * - * @return collection of service names - */ - Collection getServices(); - /** * Get the configuration associated with the host group. * The host group configuration has the blueprint cluster scoped @@ -122,13 +76,6 @@ public interface HostGroup { */ Configuration getConfiguration(); - /** - * Get the stack associated with the host group. - * - * @return associated stack - */ - StackDefinition getStack(); - /** * Get the cardinality value that was specified for the host group. * This is simply meta-data for the stack that a deployer can use diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java index ffe6a249c63..be607e7b0ba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java @@ -20,10 +20,8 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -31,7 +29,6 @@ import java.util.Set; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; @@ -46,58 +43,35 @@ public class HostGroupImpl implements HostGroup { /** * host group name */ - private String name; - - /** - * blueprint name - */ - private String blueprintName; + private final String name; /** * components contained in the host group */ private final Set components = new LinkedHashSet<>(); - /** - * map of service to components for the host group - */ - // TODO: in blueprint 3.0 this should be per service instance - private Map> componentsForService = new HashMap<>(); - /** * configuration */ - private Configuration configuration = null; - - private boolean containsMasterComponent = false; + private final Configuration configuration; + private final String cardinality; - private StackDefinition stack; - - private String cardinality = "NOT SPECIFIED"; - - public HostGroupImpl(HostGroupEntity entity, String blueprintName, StackDefinition stack) { + public HostGroupImpl(HostGroupEntity entity) { this.name = entity.getName(); this.cardinality = entity.getCardinality(); - this.blueprintName = blueprintName; - this.stack = stack; + configuration = parseConfigurations(entity); parseComponents(entity); - parseConfigurations(entity); } - public HostGroupImpl(String name, String bpName, StackDefinition stack, Collection components, Configuration configuration, String cardinality) { + public HostGroupImpl(String name, Collection components, Configuration configuration, String cardinality) { this.name = name; - this.blueprintName = bpName; - this.stack = stack; + this.configuration = configuration; + this.cardinality = (cardinality != null && !"null".equals(cardinality)) ? cardinality : "NOT SPECIFIED"; // process each component for (Component component: components) { addComponent(component); } - - this.configuration = configuration; - if (cardinality != null && ! cardinality.equals("null")) { - this.cardinality = cardinality; - } } @@ -129,68 +103,15 @@ public Collection getComponentNames(ProvisionAction provisionAction) { return setOfComponentNames; } - /** - * Get the services which are deployed to this host group. - * - * @return collection of services which have components in this host group - */ - @Override - public Collection getServices() { - return componentsForService.keySet(); - } - /** * Adds a component to the host group. * @param component the component to add */ @Override public boolean addComponent(Component component) { - if (components.add(component)) { - containsMasterComponent |= stack.isMasterComponent(component.getName()); - - String service = stack.getServiceForComponent(component.getName()); - if (service != null) { - componentsForService - .computeIfAbsent(service, __ -> new HashSet<>()) - .add(component); - } - - return true; - } - - return false; - } - - /** - * Get the components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of components - */ - @Override - public Collection getComponents(String service) { - return componentsForService.containsKey(service) ? - new HashSet<>(componentsForService.get(service)) : - Collections.emptySet(); - } - - /** - * Get the names of components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of component names - */ - @Override - @Deprecated - public Collection getComponentNames(String service) { - return componentsForService.containsKey(service) ? - componentsForService.get(service).stream().map(Component::getName).collect(toSet()) : - Collections.emptySet(); + return components.add(component); } - /** * Get this host groups configuration. * @@ -201,26 +122,6 @@ public Configuration getConfiguration() { return configuration; } - /** - * Get the associated blueprint name. - * - * @return associated blueprint name - */ - @Override - public String getBlueprintName() { - return blueprintName; - } - - @Override - public boolean containsMasterComponent() { - return containsMasterComponent; - } - - @Override - public StackDefinition getStack() { - return stack; - } - @Override public String getCardinality() { return cardinality; @@ -244,16 +145,12 @@ private void parseComponents(HostGroupEntity entity) { * Parse host group configurations. */ //todo: use ConfigurationFactory - private void parseConfigurations(HostGroupEntity entity) { + private Configuration parseConfigurations(HostGroupEntity entity) { Map> config = new HashMap<>(); Gson jsonSerializer = new Gson(); for (HostGroupConfigEntity configEntity : entity.getConfigurations()) { String type = configEntity.getType(); - Map typeProperties = config.get(type); - if (typeProperties == null) { - typeProperties = new HashMap<>(); - config.put(type, typeProperties); - } + Map typeProperties = config.computeIfAbsent(type, k -> new HashMap<>()); Map propertyMap = jsonSerializer.>fromJson( configEntity.getConfigData(), Map.class); if (propertyMap != null) { @@ -262,9 +159,10 @@ private void parseConfigurations(HostGroupEntity entity) { } //todo: parse attributes Map>> attributes = new HashMap<>(); - configuration = new Configuration(config, attributes); + return new Configuration(config, attributes); } + @Override public String toString(){ return name; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java index 3a8acc82162..5e3874b5913 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java @@ -98,7 +98,7 @@ public HostRequest(long requestId, long id, long clusterId, String hostname, Str this.hostGroup = hostGroup; hostgroupName = hostGroup.getName(); this.predicate = predicate; - containsMaster = hostGroup.containsMasterComponent(); + containsMaster = topology.containsMasterComponent(hostGroup); this.topology = topology; this.skipFailure = skipFailure; createTasks(this.skipFailure); @@ -128,7 +128,7 @@ public HostRequest(long requestId, long id, String predicate, setStatus(entity.getStatus()); statusMessage = entity.getStatusMessage(); this.predicate = toPredicate(predicate); - containsMaster = hostGroup.containsMasterComponent(); + containsMaster = topology.containsMasterComponent(hostGroup); this.topology = topology; this.skipFailure = skipFailure; @@ -257,7 +257,7 @@ private void createTasks(boolean skipFailure) { "PENDING HOST ASSIGNMENT : HOSTGROUP=" + getHostgroupName(); AmbariContext context = topology.getAmbariContext(); - StackDefinition stack = hostGroup.getStack(); + StackDefinition stack = topology.getStack(); // Skip INSTALL task in case server component is marked as START_ONLY, or the cluster provision_action is // START_ONLY, unless component is marked with INSTALL_ONLY or INSTALL_AND_START. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java index b9bbe2cf12c..b23aa25e649 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java @@ -444,7 +444,7 @@ public Optional getFailureReason() { private void createHostRequests(TopologyRequest request, ClusterTopology topology) { Map hostGroupInfoMap = request.getHostGroupInfo(); Blueprint blueprint = topology.getBlueprint(); - boolean skipFailure = topology.getBlueprint().shouldSkipFailure(); + boolean skipFailure = blueprint.getSetting().shouldSkipFailure(); for (HostGroupInfo hostGroupInfo : hostGroupInfoMap.values()) { String groupName = hostGroupInfo.getHostGroupName(); int hostCardinality = hostGroupInfo.getRequestedHostCount(); @@ -495,7 +495,7 @@ private void createHostRequests(ClusterTopology topology, } } - boolean skipFailure = topology.getBlueprint().shouldSkipFailure(); + boolean skipFailure = topology.getBlueprint().getSetting().shouldSkipFailure(); for (TopologyHostRequestEntity hostRequestEntity : requestEntity.getTopologyHostRequestEntities()) { Long hostRequestId = hostRequestEntity.getId(); synchronized (hostIdCounter) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java index 3a09af119f0..cf134735605 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java @@ -149,7 +149,7 @@ private String getString(String outerKey, String innerKey) { return null; } - boolean shouldSkipFailure() { + public boolean shouldSkipFailure() { return Boolean.parseBoolean(getString(SETTING_NAME_DEPLOYMENT_SETTINGS, SETTING_NAME_SKIP_FAILURE)); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 51eedd31e23..25a56d405d9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -130,6 +130,9 @@ public class TopologyManager { //todo: currently only support a single cluster private Map clusterTopologyMap = new HashMap<>(); + @Inject + private Configuration configuration; + @Inject private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor; @@ -277,7 +280,7 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request); final String clusterName = request.getClusterName(); - final StackDefinition stack = topology.getBlueprint().getStack(); + final StackDefinition stack = topology.getStack(); final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); @@ -294,7 +297,7 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ addKerberosClient(topology); // refresh default stack config after adding KERBEROS_CLIENT component to topology - topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint().getServices())); + topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getServices())); credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); if (credential == null) { @@ -304,7 +307,6 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ topologyValidatorService.validateTopologyConfiguration(topology); - // create resources ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java index ff79f526ea6..44f0d1f02ac 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java @@ -27,6 +27,7 @@ * Factory for creating topology requests. */ public class TopologyRequestFactoryImpl implements TopologyRequestFactory { + @Override public ProvisionClusterRequest createProvisionClusterRequest(Map properties, SecurityConfiguration securityConfiguration) throws InvalidTopologyTemplateException { return new ProvisionClusterRequest(properties, securityConfiguration); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java index 74fb9b1c897..58f174c3209 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java @@ -17,13 +17,7 @@ */ package org.apache.ambari.server.topology.tasks; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,16 +41,10 @@ public Type getType() { @Override public void runTask() { - LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostRequest.getHostName()); - - HostGroup group = hostRequest.getHostGroup(); - Map> serviceComponents = new HashMap<>(); - for (String service : group.getServices()) { - serviceComponents.put(service, new HashSet<>(group.getComponentNames(service))); - } - clusterTopology.getAmbariContext().createAmbariHostResources(hostRequest.getClusterId(), - hostRequest.getHostName(), serviceComponents); - - LOG.info("HostRequest: Exiting RESOURCE_CREATION task for host: {}", hostRequest.getHostName()); + String hostName = hostRequest.getHostName(); + LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostName); + long clusterId = hostRequest.getClusterId(); + clusterTopology.getAmbariContext().createAmbariHostResources(clusterId, hostName, hostRequest.getHostGroup().getComponents()); + LOG.info("HostRequest: Exiting RESOURCE_CREATION task for host: {}", hostName); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java new file mode 100644 index 00000000000..82ea759f457 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.Blueprint; + +public class BasicBlueprintValidator implements BlueprintValidator { + + @Override + public void validate(Blueprint blueprint) throws IllegalArgumentException { + // FIXME implement + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java similarity index 51% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java index 64e08ad1b5c..53d58785a8b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java @@ -16,27 +16,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.Blueprint; /** - * Provides blueprint validation. + * Provides basic blueprint validation. */ public interface BlueprintValidator { - /** - * Validate blueprint topology. - * - * @param blueprint the blueprint to validate - * @throws InvalidTopologyException if the topology is invalid - */ - void validateTopology(Blueprint blueprint) throws InvalidTopologyException; - /** - * Validate that required properties are provided. - * This doesn't include password properties. - * - * @param blueprint the blueprint to validate - * @throws InvalidTopologyException if required properties are not set in blueprint - * @throws GPLLicenseNotAcceptedException if the blueprint requires use of GPL software, but GPL license was not accepted - */ - void validateRequiredProperties(Blueprint blueprint) throws InvalidTopologyException, GPLLicenseNotAcceptedException; + void validate(Blueprint blueprint) throws IllegalArgumentException; + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java index a65ea6dcde4..25748695943 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java @@ -43,8 +43,8 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { // collecting all config types for services in the blueprint (from the related stack) Set stackServiceConfigTypes = new HashSet<>(); - for (String serviceName : topology.getBlueprint().getServices()) { - stackServiceConfigTypes.addAll(topology.getBlueprint().getStack().getConfigurationTypes(serviceName)); + for (String serviceName : topology.getServices()) { + stackServiceConfigTypes.addAll(topology.getStack().getConfigurationTypes(serviceName)); } // identifying invalid config types diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java similarity index 76% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java index 1e7f0ec0844..d4b931c8075 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java @@ -16,63 +16,57 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.utils.SecretReference; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.ClusterTopologyImpl; +import org.apache.ambari.server.topology.Component; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; - /** - * Default blueprint validator. + * Verifies that service dependencies and component cardinality requirements are satisfied. */ -public class BlueprintValidatorImpl implements BlueprintValidator { - - private static final Logger LOGGER = LoggerFactory.getLogger(BlueprintValidatorImpl.class); - - public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class"; - public static final String CODEC_CLASSES_PROPERTY_NAME = "io.compression.codecs"; - public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec"; +public class DependencyAndCardinalityValidator implements TopologyValidator { - private final Configuration configuration; - - @Inject - public BlueprintValidatorImpl(Configuration configuration) { - this.configuration = configuration; - } + private static final Logger LOGGER = LoggerFactory.getLogger(DependencyAndCardinalityValidator.class); @Override - public void validateTopology(Blueprint blueprint) throws InvalidTopologyException { + public void validate(ClusterTopology topology) throws InvalidTopologyException { + Blueprint blueprint = topology.getBlueprint(); LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName()); - StackDefinition stack = blueprint.getStack(); + StackDefinition stack = topology.getStack(); Collection hostGroups = blueprint.getHostGroups().values(); Map>> missingDependencies = new HashMap<>(); for (HostGroup group : hostGroups) { - Map> missingGroupDependencies = validateHostGroup(blueprint, stack, group); + Map> missingGroupDependencies = validateHostGroup(topology, blueprint, stack, group); if (!missingGroupDependencies.isEmpty()) { missingDependencies.put(group.getName(), missingGroupDependencies); } } Collection cardinalityFailures = new HashSet<>(); - Collection services = blueprint.getServices(); + Collection services = topology.getServices(); for (String service : services) { for (String component : stack.getComponents(service)) { @@ -92,68 +86,19 @@ public void validateTopology(Blueprint blueprint) throws InvalidTopologyExceptio } } - @Override - public void validateRequiredProperties(Blueprint blueprint) throws InvalidTopologyException, GPLLicenseNotAcceptedException { - + public void validateRequiredProperties(ClusterTopology topology) throws InvalidTopologyException { // we don't want to include default stack properties so we can't just use hostGroup full properties - Map> clusterConfigurations = blueprint.getConfiguration().getProperties(); - - // we need to have real passwords, not references - if (clusterConfigurations != null) { - - // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it - boolean gplEnabled = configuration.getGplLicenseAccepted(); + Map> clusterConfigurations = topology.getConfiguration().getProperties(); - StringBuilder errorMessage = new StringBuilder(); - boolean containsSecretReferences = false; - for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { - String configType = configEntry.getKey(); - if (configEntry.getValue() != null) { - for (Map.Entry propertyEntry : configEntry.getValue().entrySet()) { - String propertyName = propertyEntry.getKey(); - String propertyValue = propertyEntry.getValue(); - if (propertyValue != null) { - if (!gplEnabled && configType.equals("core-site") - && (propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME) || propertyName.equals(CODEC_CLASSES_PROPERTY_NAME)) - && propertyValue.contains(LZO_CODEC_CLASS)) { - throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " + - "Please refer to documentation to configure Ambari before proceeding."); - } - if (SecretReference.isSecret(propertyValue)) { - errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n"); - containsSecretReferences = true; - } - } - } - } - } - if (containsSecretReferences) { - throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + - "replace following properties with real passwords:\n" + errorMessage); - } - } - - - for (HostGroup hostGroup : blueprint.getHostGroups().values()) { - Collection processedServices = new HashSet<>(); - Map> allRequiredProperties = new HashMap<>(); + for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { Map> operationalConfiguration = new HashMap<>(clusterConfigurations); operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); for (String component : hostGroup.getComponentNames()) { - //check that MYSQL_SERVER component is not available while hive is using existing db - if (component.equals("MYSQL_SERVER")) { - Map hiveEnvConfig = clusterConfigurations.get("hive-env"); - if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null - && hiveEnvConfig.get("hive_database").startsWith("Existing")) { - throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + - " using existing db!"); - } - } if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { Map hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - ArrayList hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component)); + List hostGroupsForComponent = new ArrayList<>(topology.getBlueprint().getHostGroupsForComponent(component)); Set givenHostGroups = new HashSet<>(); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); @@ -207,7 +152,7 @@ private Collection verifyComponentInAllHostGroups(Blueprint blueprint, C return cardinalityFailures; } - private Map> validateHostGroup(Blueprint blueprint, StackDefinition stack, HostGroup group) { + private Map> validateHostGroup(ClusterTopology topology, Blueprint blueprint, StackDefinition stack, HostGroup group) { LOGGER.info("Validating hostgroup: {}", group.getName()); Map> missingDependencies = new HashMap<>(); @@ -219,9 +164,9 @@ private Map> validateHostGroup(Blueprint blue // dependent components from the stack definitions are only added if related services are explicitly added to the blueprint! boolean isClientDependency = stack.getComponentInfo(dependency.getComponentName()).isClient(); - if (isClientDependency && !blueprint.getServices().contains(dependency.getServiceName())) { + if (isClientDependency && !topology.getServices().contains(dependency.getServiceName())) { LOGGER.debug("The service [{}] for component [{}] is missing from the blueprint [{}], skipping dependency", - dependency.getServiceName(), dependency.getComponentName(), blueprint.getName()); + dependency.getServiceName(), dependency.getComponentName(), topology.getBlueprint().getName()); continue; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java new file mode 100644 index 00000000000..467908e648c --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableSet; + +/** + * Properties that indicate usage of GPL software are + * allowed with explicit approval from user. + */ +public class GplPropertiesValidator implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(GplPropertiesValidator.class); + + private static final String CORE_SITE = "core-site"; + static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class"; + static final String CODEC_CLASSES_PROPERTY_NAME = "io.compression.codecs"; + static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec"; + + private static final Set PROPERTY_NAMES = ImmutableSet.of( + LZO_CODEC_CLASS_PROPERTY_NAME, + CODEC_CLASSES_PROPERTY_NAME + ); + + private static final String GPL_LICENSE_ERROR_MESSAGE = + "Your Ambari server has not been configured to download LZO GPL software. " + + "Please refer to documentation to configure Ambari before proceeding."; + + private final Configuration configuration; + + @Inject + public GplPropertiesValidator(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it + boolean gplEnabled = configuration.getGplLicenseAccepted(); + + if (gplEnabled) { + LOG.info("GPL license accepted, skipping config check"); + return; + } + + // we don't want to include default stack properties so we can't use full properties + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + if (clusterConfigurations != null) { + for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { + String configType = configEntry.getKey(); + if (!CORE_SITE.equals(configType)) { + continue; + } + + Map properties = configEntry.getValue(); + if (properties == null) { + continue; + } + + for (String propertyName : PROPERTY_NAMES) { + String propertyValue = properties.get(propertyName); + if (propertyValue != null && propertyValue.contains(LZO_CODEC_CLASS)) { + throw new GPLLicenseNotAcceptedException(GPL_LICENSE_ERROR_MESSAGE); + } + } + } + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index bc9918ad8c5..1a9bf07dda3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -30,14 +30,13 @@ public class HiveServiceValidator implements TopologyValidator { private static final String HIVE_DB_DEFAULT = "New MySQL Database"; private static final String HIVE_DB_PROPERTY = "hive_database"; private static final String MYSQL_SERVER_COMPONENT = "MYSQL_SERVER"; - public static final String HIVE_SERVICE = "HIVE"; + private static final String HIVE_SERVICE = "HIVE"; @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { - // there is no hive configured in the blueprint, nothing to do (does the validator apply?) - if (!topology.getBlueprint().getServices().contains(HIVE_SERVICE)) { + if (!topology.getServices().contains(HIVE_SERVICE)) { LOGGER.info(" [{}] service is not listed in the blueprint, skipping hive service validation.", HIVE_SERVICE); return; } @@ -51,16 +50,21 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { throw new InvalidTopologyException(errorMessage); } - // hive database has custom configuration, skipping validation - if (!HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY))) { - LOGGER.info("Custom hive database settings detected. HIVE service validation succeeded."); - return; + boolean hiveWantsMysql = HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY)); + boolean topologyContainsMysql = topology.getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT); + + if (topologyContainsMysql && !hiveWantsMysql) { + String errorMessage = String.format( + "Incorrect configuration: %s component is specified in blueprint, but Hive is configured to use existing DB", + MYSQL_SERVER_COMPONENT); + LOGGER.error(errorMessage); + throw new InvalidTopologyException(errorMessage); } - // hive database settings need the mysql-server component in the blueprint - if (!topology.getBlueprint().getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) { - String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " + - "is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT); + if (hiveWantsMysql && !topologyContainsMysql) { + String errorMessage = String.format( + "The component %s must explicitly be specified in the blueprint if Hive database is configured with %s.", + MYSQL_SERVER_COMPONENT, HIVE_DB_DEFAULT); LOGGER.error(errorMessage); throw new InvalidTopologyException(errorMessage); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java new file mode 100644 index 00000000000..1fa043c0eb8 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology.validators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.ClusterTopologyImpl; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Verifies that dfs_ha_initial_namenode_active and dfs_ha_initial_namenode_standby properties + * in hadoop-env reference host groups with NAMENODE components. + */ +public class NameNodeHighAvailabilityValidator implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(NameNodeHighAvailabilityValidator.class); + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + Blueprint blueprint = topology.getBlueprint(); + + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + if (!ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations)) { + LOG.info("NAMENODE HA is not enabled, skipping validation of {}", blueprint.getName()); + return; + } + + LOG.info("Validating NAMENODE HA for blueprint: {}", blueprint.getName()); + + List hostGroupsForComponent = new ArrayList<>(topology.getBlueprint().getHostGroupsForComponent("NAMENODE")); + + for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { + Map> operationalConfiguration = new HashMap<>(clusterConfigurations); + + operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); + if (hostGroup.getComponentNames().contains("NAMENODE")) { + Map hadoopEnvConfig = operationalConfiguration.get("hadoop-env"); + if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { + Set givenHostGroups = new HashSet<>(); + givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); + givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); + if(givenHostGroups.size() != hostGroupsForComponent.size()) { + throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); + } + if (BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()) { + for (HostGroup hostGroupForComponent : hostGroupsForComponent) { + Iterator itr = givenHostGroups.iterator(); + while(itr.hasNext()){ + if(itr.next().contains(hostGroupForComponent.getName())){ + itr.remove(); + } + } + } + } + + if(!givenHostGroups.isEmpty()){ + throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); + } + } + } + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java new file mode 100644 index 00000000000..6a9706f3581 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.joining; + +import org.apache.ambari.server.controller.RootComponent; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RejectUnknownComponents implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(RejectUnknownComponents.class); + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + String unknownComponents = topology.getComponentNames() + .filter(c -> !RootComponent.AMBARI_SERVER.name().equals(c)) + .filter(c -> !topology.getStack().getComponents().contains(c)) + .collect(joining(", ")); + + if (!unknownComponents.isEmpty()) { + String msg = "The following components are not valid for the specified stacks: " + unknownComponents; + LOG.info(msg); + throw new InvalidTopologyException(msg); + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java index 7b1a604d99e..5102fb97a74 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java @@ -23,7 +23,6 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.InvalidTopologyException; @@ -43,7 +42,7 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator { /** * Validates the configuration coming from the blueprint and cluster creation template and ensures that all the required properties are provided. - * It's expected, that a in hostrgroup containing components for a given service all required configuration for the given service is available. + * It's expected, that a in hostgroup containing components for a given service all required configuration for the given service is available. * * @param topology the topology instance holding the configuration for cluster provisioning * @throws InvalidTopologyException when there are missing configuration types or properties related to services in the blueprint @@ -52,36 +51,30 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator { public void validate(ClusterTopology topology) throws InvalidTopologyException { // collect required properties - Map>> requiredPropertiesByService = getRequiredPropertiesByService(topology.getBlueprint()); + Map>> requiredPropertiesByService = getRequiredPropertiesByService(topology); // find missing properties in the cluster configuration Map> missingProperties = new TreeMap<>(); Map> topologyConfiguration = new HashMap<>(topology.getConfiguration().getFullProperties(1)); - for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { - LOGGER.debug("Processing hostgroup configurations for hostgroup: {}", hostGroup.getName()); + for (Map.Entry>> entry : requiredPropertiesByService.entrySet()) { + String service = entry.getKey(); + Map> requiredPropertiesByType = entry.getValue(); - // copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig - Map> operationalConfigurations = new HashMap<>(topologyConfiguration); + for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { + LOGGER.debug("Processing configurations for service {} in hostgroup {}", service, hostGroup.getName()); - for (Map.Entry> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) { - if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) { - operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue()); - } else { - operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue()); - } - } - - for (String hostGroupService : hostGroup.getServices()) { + // copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig + Map> operationalConfigurations = new HashMap<>(topologyConfiguration); - if (!requiredPropertiesByService.containsKey(hostGroupService)) { - // there are no required properties for the service - LOGGER.debug("There are no required properties found for hostgroup/service: [{}/{}]", hostGroup.getName(), hostGroupService); - continue; + for (Map.Entry> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) { + if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) { + operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue()); + } else { + operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue()); + } } - Map> requiredPropertiesByType = requiredPropertiesByService.get(hostGroupService); - for (String configType : requiredPropertiesByType.keySet()) { // We need a copy not to modify the original @@ -117,18 +110,17 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { * Collects required properties for services in the blueprint. Configuration properties are returned by configuration type. * service -> configType -> properties * - * @param blueprint the blueprint from the cluster topology * @return a map with configuration types mapped to collections of required property names */ - private Map>> getRequiredPropertiesByService(Blueprint blueprint) { + private Map>> getRequiredPropertiesByService(ClusterTopology topology) { Map>> requiredPropertiesForServiceByType = new HashMap<>(); - for (String bpService : blueprint.getServices()) { + for (String bpService : topology.getServices()) { LOGGER.debug("Collecting required properties for the service: {}", bpService); - Collection requiredConfigsForService = blueprint.getStack().getRequiredConfigurationProperties(bpService); + Collection requiredConfigsForService = topology.getStack().getRequiredConfigurationProperties(bpService); Map> requiredPropertiesByConfigType = new HashMap<>(); for (Stack.ConfigProperty configProperty : requiredConfigsForService) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index 3db6948674c..e79dc407463 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -82,7 +82,7 @@ private Map>> validateRequiredPasswords(C Collection processedServices = new HashSet<>(); Blueprint blueprint = topology.getBlueprint(); - StackDefinition stack = blueprint.getStack(); + StackDefinition stack = topology.getStack(); HostGroup hostGroup = blueprint.getHostGroup(hostGroupName); for (String component : hostGroup.getComponentNames()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java new file mode 100644 index 00000000000..2e3c61b127e --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import java.util.Map; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.utils.SecretReference; + +/** + * Secret references are not allowed in blueprints. + * @see SecretReference + */ +public class SecretReferenceValidator implements TopologyValidator { + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + // we don't want to include default stack properties so we can't use full properties + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + // we need to have real passwords, not references + if (clusterConfigurations != null) { + StringBuilder errorMessage = new StringBuilder(); + boolean containsSecretReferences = false; + for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { + String configType = configEntry.getKey(); + Map configEntryValue = configEntry.getValue(); + if (configEntryValue != null) { + for (Map.Entry propertyEntry : configEntryValue.entrySet()) { + String propertyName = propertyEntry.getKey(); + String propertyValue = propertyEntry.getValue(); + if (propertyValue != null && SecretReference.isSecret(propertyValue)) { + errorMessage.append(String.format(" Config:%s Property:%s\n", configType, propertyName)); + containsSecretReferences = true; + } + } + } + } + if (containsSecretReferences) { + throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + + "replace following properties with real passwords:\n" + errorMessage); + } + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java index 4f51639e2d5..7166998b016 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java @@ -44,7 +44,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { return; } - Set stackConfigTypes = new HashSet<>(topology.getBlueprint().getStack().getConfiguration().getAllConfigTypes()); + Set stackConfigTypes = new HashSet<>(topology.getStack().getConfiguration().getAllConfigTypes()); // remove all "valid" config types from the incoming set incomingConfigTypes.removeAll(stackConfigTypes); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index de2c39d4152..1e81ef84b97 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -20,6 +20,7 @@ import javax.inject.Provider; import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; import com.google.common.collect.ImmutableList; @@ -28,13 +29,18 @@ public class TopologyValidatorFactory { private final List validators; @Inject - public TopologyValidatorFactory(Provider metaInfo) { + public TopologyValidatorFactory(Provider metaInfo, Configuration config) { validators = ImmutableList.builder() .add(new RejectUnknownStacks(metaInfo)) + .add(new RejectUnknownComponents()) + .add(new DependencyAndCardinalityValidator()) + .add(new StackConfigTypeValidator()) + .add(new GplPropertiesValidator(config)) + .add(new SecretReferenceValidator()) .add(new RequiredConfigPropertiesValidator()) .add(new RequiredPasswordValidator()) .add(new HiveServiceValidator()) - .add(new StackConfigTypeValidator()) + .add(new NameNodeHighAvailabilityValidator()) .add(new UnitValidator(UnitValidatedProperty.ALL)) .build(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java index 94f28f102e5..18e3913361a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java @@ -41,7 +41,7 @@ public UnitValidator(Set propertiesToBeValidated) { @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); validateConfig(topology.getConfiguration().getFullProperties(), stack); for (HostGroupInfo hostGroup : topology.getHostGroupInfo().values()) { validateConfig(hostGroup.getConfiguration().getFullProperties(), stack); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java index 7a1ce8b2175..6642f58976d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java @@ -167,7 +167,7 @@ public void setup() throws Exception { expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); expect(blueprint.getHostGroup("host_group_1")).andReturn(group1).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java index 10fde6fa39f..4fa28acafca 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java @@ -87,16 +87,16 @@ public void testAdviseConfiguration() throws StackAdvisorException, Configuratio expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -126,16 +126,16 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApply() throws StackAdvi expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -166,16 +166,16 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApplyWhenNoUserInputForD expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -204,16 +204,16 @@ public void testAdviseConfigurationWith_ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -240,7 +240,7 @@ public void testAdviseConfigurationWhenConfigurationRecommendFails() throws Stac expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes(); expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); @@ -248,7 +248,7 @@ public void testAdviseConfigurationWhenConfigurationRecommendFails() throws Stac expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andThrow(new StackAdvisorException("ex")); expect(configuration.getFullProperties()).andReturn(props); @@ -273,14 +273,14 @@ public void testAdviseConfigurationWhenConfigurationRecommendHasInvalidResponse( expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes(); expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(new RecommendationResponse()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 23283702dfa..99096438b5b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.controller.internal; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -144,7 +145,7 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { @Before public void init() throws Exception { - expect(bp.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(bp.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(bp.getName()).andReturn("test-bp").anyTimes(); @@ -4275,10 +4276,10 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.emptySet()); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4318,10 +4319,10 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4358,8 +4359,8 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); Map> properties = new HashMap<>(); Map typeProps = new HashMap<>(); @@ -4407,10 +4408,10 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); // simulate the case where the STORM service has been removed manually from the stack definitions expect(stack.getServiceForConfigType("storm-site")).andThrow(new IllegalArgumentException("TEST: Configuration not found in stack definitions!")); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -6329,7 +6330,6 @@ public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception Collection hgComponents = new HashSet<>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); - hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection hgComponents2 = new HashSet<>(); @@ -6355,7 +6355,8 @@ public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception reset(stack); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); - expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); + expect(stack.getServiceForComponent(anyString())).andReturn("HDFS").anyTimes(); + expect(stack.getConfiguration(ImmutableSet.of("HDFS"))).andReturn(createStackDefaults()).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -6387,7 +6388,6 @@ public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly Collection hgComponents = new HashSet<>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); - hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection hgComponents2 = new HashSet<>(); @@ -6415,7 +6415,8 @@ public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly reset(stack); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); - expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); + expect(stack.getServiceForComponent(anyString())).andReturn("HDFS").anyTimes(); + expect(stack.getConfiguration(ImmutableSet.of("HDFS"))).andReturn(createStackDefaults()).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -8219,7 +8220,6 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters, cluster); Map hostGroupInfo = new HashMap<>(); - Collection allServices = new HashSet<>(); Map allHostGroups = new HashMap<>(); for (TestHostGroup hostGroup : hostGroups) { @@ -8230,26 +8230,15 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration List componentList = new ArrayList<>(); for (String componentName : hostGroup.components) { - componentList.add(new Component(componentName)); + componentList.add(new Component(componentName, null, stack.getServiceForComponent(componentName), null)); } //create host group which is set on topology - allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", stack, - componentList, EMPTY_CONFIG, "1")); + allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, componentList, EMPTY_CONFIG, "1")); hostGroupInfo.put(hostGroup.name, groupInfo); - - for (String component : hostGroup.components) { - for (Map.Entry> serviceComponentsEntry : serviceComponents.entrySet()) { - if (serviceComponentsEntry.getValue().contains(component)) { - allServices.add(serviceComponentsEntry.getKey()); - } - } - } } - expect(bp.getServices()).andReturn(allServices).anyTimes(); - for (HostGroup group : allHostGroups.values()) { expect(bp.getHostGroup(group.getName())).andReturn(group).anyTimes(); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index 5ee048d6dfc..aeadc5cb74a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -75,11 +75,10 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.ambari.server.topology.Setting; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.StageUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -90,7 +89,6 @@ /** * BlueprintResourceProvider unit tests. */ -@SuppressWarnings("unchecked") public class BlueprintResourceProviderTest { private static String BLUEPRINT_NAME = "test-blueprint"; @@ -138,8 +136,7 @@ public void testCreateResources() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); expect(securityFactory.createSecurityConfigurationFromRequest(null, true)).andReturn(null).anyTimes(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -215,7 +212,6 @@ public void testCreateResources_NoValidation() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -247,7 +243,7 @@ public void testCreateResources_NoValidation() throws Exception { verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request, managementController); } - @Test + @Test(expected = IllegalArgumentException.class) public void testCreateResources_TopologyValidationFails() throws Exception { Request request = createMock(Request.class); @@ -257,9 +253,8 @@ public void testCreateResources_TopologyValidationFails() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); - expectLastCall().andThrow(new InvalidTopologyException("test")).once(); + blueprintValidator.validate(blueprint); + expectLastCall().andThrow(new IllegalArgumentException("test")).once(); expect(request.getProperties()).andReturn(setProperties); expect(request.getRequestInfoProperties()).andReturn(requestInfoProperties); @@ -275,14 +270,7 @@ public void testCreateResources_TopologyValidationFails() throws Exception { AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); ((ObservableResourceProvider)provider).addObserver(observer); - try { - provider.createResources(request); - fail("Expected exception due to topology validation error"); - } catch (IllegalArgumentException e) { - // expected - } - - verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request); + provider.createResources(request); } @@ -299,8 +287,7 @@ public void testCreateResources_withConfiguration() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -332,9 +319,8 @@ public void testCreateResources_withConfiguration() throws Exception { verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request, managementController); } - @Test - public void testCreateResource_BlueprintFactoryThrowsException() throws Exception - { + @Test(expected = IllegalArgumentException.class) + public void testCreateResource_BlueprintFactoryThrowsException() throws Exception { Request request = createMock(Request.class); Set> setProperties = getBlueprintTestProperties(); @@ -352,13 +338,7 @@ public void testCreateResource_BlueprintFactoryThrowsException() throws Exceptio replay(dao, entity, metaInfo, blueprintFactory, securityFactory, blueprint, blueprintValidator, request); // end expectations - try { - provider.createResources(request); - fail("Exception expected"); - } catch (IllegalArgumentException e) { - // expected - } - verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request); + provider.createResources(request); } @Test @@ -376,8 +356,7 @@ public void testCreateResources_withSecurityConfiguration() throws Exception { expect(securityFactory.createSecurityConfigurationFromRequest(anyObject(), anyBoolean())).andReturn (securityConfiguration).once(); expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), securityConfiguration)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -502,8 +481,7 @@ public void testCreateResources_withEmptyConfiguration() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -549,8 +527,7 @@ public void testCreateResources_withSingleConfigurationType() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java index d8cb701da5c..440b2e81275 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java @@ -85,6 +85,7 @@ import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.HostRequest; import org.apache.ambari.server.topology.LogicalRequest; +import org.apache.ambari.server.topology.Setting; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.topology.TopologyRequest; import org.apache.ambari.server.utils.SecretReference; @@ -1684,7 +1685,9 @@ public void testGetLogicalRequestStatusWithNoTasks() throws Exception { expect(topologyRequest.getHostGroupInfo()).andReturn(hostGroupInfoMap).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.shouldSkipFailure()).andReturn(true).anyTimes(); + Setting setting = createNiceMock(Setting.class); + expect(blueprint.getSetting()).andReturn(setting).anyTimes(); + expect(setting.shouldSkipFailure()).andReturn(true).anyTimes(); expect(logicalRequest.getHostRequests()).andReturn(hostRequests).anyTimes(); expect(logicalRequest.constructNewPersistenceEntity()).andReturn(requestMock).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java index a7d0d479d2e..a08691862af 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java @@ -106,7 +106,7 @@ private void stackUnitIs(String name, String unit) { private String updateUnit(String serviceName, String configType, String propName, String propValue) throws InvalidTopologyException, ConfigurationTopologyException { UnitUpdater updater = new UnitUpdater(serviceName, configType); expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(serviceName, configType)).andReturn(stackConfigWithMetadata).anyTimes(); replayAll(); return updater.updateForClusterCreate(propName, propValue, Collections.emptyMap(), clusterTopology); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index b56835424eb..c0ce6b322b9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -137,6 +137,7 @@ public class AmbariContextTest { private static final Collection group1Hosts = Arrays.asList(HOST1, HOST2); private Capture> configGroupRequestCapture = EasyMock.newCapture(); + private Setting setting = createNiceMock(Setting.class); @Before public void setUp() throws Exception { @@ -242,15 +243,16 @@ public void setUp() throws Exception { expect(topology.getHostGroupInfo()).andReturn(Collections.singletonMap(HOST_GROUP_1, group1Info)).anyTimes(); expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); - expect(blueprint.getServices()).andReturn(blueprintServices).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); - expect(blueprint.getStackIdsForService("service1")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.getStackIdsForService("service2")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getServices()).andReturn(blueprintServices).anyTimes(); + expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); + expect(topology.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); + expect(topology.getStackIdsForService("service1")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getStackIdsForService("service2")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); - expect(blueprint.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); + expect(blueprint.getSetting()).andReturn(setting).anyTimes(); + expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); @@ -287,13 +289,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } private void replayAll() { replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } @@ -407,16 +409,12 @@ public void testCreateAmbariHostResources() throws Exception { replayAll(); // test - Map> componentsMap = new HashMap<>(); - Collection components = new ArrayList<>(); - components.add("component1"); - components.add("component2"); - componentsMap.put("service1", components); - components = new ArrayList<>(); - components.add("component3"); - componentsMap.put("service2", components); + Collection components = new ArrayList<>(); + components.add(new Component("component1", "mpack", "service1", null)); + components.add(new Component("component2", "mpack", "service1", null)); + components.add(new Component("component3", "mpack", "service2", null)); - context.createAmbariHostResources(CLUSTER_ID, "host1", componentsMap); + context.createAmbariHostResources(CLUSTER_ID, "host1", components); assertEquals(requestsCapture.getValue().size(), 3); } @@ -436,16 +434,12 @@ public void testCreateAmbariHostResourcesWithMissingService() throws Exception { replayAll(); // test - Map> componentsMap = new HashMap<>(); - Collection components = new ArrayList<>(); - components.add("component1"); - components.add("component2"); - componentsMap.put("service1", components); - components = new ArrayList<>(); - components.add("component3"); - componentsMap.put("service2", components); - - context.createAmbariHostResources(CLUSTER_ID, "host1", componentsMap); + Collection components = new ArrayList<>(); + components.add(new Component("component1", "mpack", "service1", null)); + components.add(new Component("component2", "mpack", "service1", null)); + components.add(new Component("component3", "mpack", "service2", null)); + + context.createAmbariHostResources(CLUSTER_ID, "host1", components); assertEquals(requestsCapture.getValue().size(), 2); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index 053cc0522ab..03791879b5c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -151,11 +151,6 @@ public void testCreateBlueprint() throws Exception { assertEquals(2, components.size()); assertTrue(components.contains("component1")); assertTrue(components.contains("component2")); - Collection services = group1.getServices(); - assertEquals(2, services.size()); - assertTrue(services.contains("test-service1")); - assertTrue(services.contains("test-service2")); - assertTrue(group1.containsMasterComponent()); //todo: add configurations/attributes to properties Configuration configuration = group1.getConfiguration(); assertTrue(configuration.getProperties().isEmpty()); @@ -167,10 +162,6 @@ public void testCreateBlueprint() throws Exception { components = group2.getComponentNames(); assertEquals(1, components.size()); assertTrue(components.contains("component1")); - services = group2.getServices(); - assertEquals(1, services.size()); - assertTrue(services.contains("test-service1")); - assertTrue(group2.containsMasterComponent()); //todo: add configurations/attributes to properties //todo: test both v1 and v2 config syntax configuration = group2.getConfiguration(); @@ -259,17 +250,6 @@ public void testCreate_HostGroupWithNoComponents() throws Exception { testFactory.createBlueprint(props, null); } - @Test(expected=IllegalArgumentException.class) - public void testCreate_HostGroupWithInvalidComponent() throws Exception { - Map props = BlueprintResourceProviderTest.getBlueprintTestProperties().iterator().next(); - // change a component name to an invalid name - ((Set>) ((Set>) props.get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). - iterator().next().get(BlueprintResourceProvider.COMPONENT_PROPERTY_ID)).iterator().next().put("name", "INVALID_COMPONENT"); - - replay(stack, dao, entity, configEntity); - testFactory.createBlueprint(props, null); - } - @Test(expected = IllegalArgumentException.class) // THEN public void verifyDefinitionsDisjointShouldRejectDuplication() { // GIVEN diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index 0019d48965d..19827e70432 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -23,7 +23,6 @@ import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertTrue; @@ -38,10 +37,9 @@ import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.state.SecurityType; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; -import com.google.common.collect.ImmutableMap; - /** * Blueprint unit tests. @@ -60,10 +58,9 @@ public class BlueprintImplTest { Map hdfsProps = new HashMap<>(); Configuration configuration = new Configuration(properties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); private final org.apache.ambari.server.configuration.Configuration serverConfig = createNiceMock(org.apache.ambari.server.configuration.Configuration.class); - private final BlueprintValidator blueprintValidator = new BlueprintValidatorImpl(serverConfig); @Before - public void setup() throws NoSuchFieldException, IllegalAccessException { + public void setup() { properties.put("hdfs-site", hdfsProps); hdfsProps.put("foo", "val"); hdfsProps.put("bar", "val"); @@ -104,12 +101,11 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null)); expect(stack.getRequiredConfigurationProperties("HDFS")).andReturn(requiredHDFSProperties).anyTimes(); expect(stack.getRequiredConfigurationProperties("SERVICE2")).andReturn(requiredService2Properties).anyTimes(); - - setupConfigurationWithGPLLicense(true); } @Test public void testValidateConfigurations__basic_positive() throws Exception { + // GIVEN expect(group1.getCardinality()).andReturn("1").atLeastOnce(); expect(group1.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c2"))).atLeastOnce(); expect(group2.getCardinality()).andReturn("1").atLeastOnce(); @@ -123,10 +119,12 @@ public void testValidateConfigurations__basic_positive() throws Exception { category2Props.put("prop2", "val"); SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, securityConfiguration, null); - blueprintValidator.validateRequiredProperties(blueprint); + + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, securityConfiguration, null); BlueprintEntity entity = blueprint.toEntity(); + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.KERBEROS); assertTrue(entity.getSecurityDescriptorReference().equals("testRef")); @@ -134,6 +132,7 @@ public void testValidateConfigurations__basic_positive() throws Exception { @Test public void testValidateConfigurations__hostGroupConfig() throws Exception { + // GIVEN Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); group2Props.put("category2", group2Category2Props); @@ -161,13 +160,18 @@ public void testValidateConfigurations__hostGroupConfig() throws Exception { hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); + + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); BlueprintEntity entity = blueprint.toEntity(); + + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.NONE); assertTrue(entity.getSecurityDescriptorReference() == null); } + + @Ignore // move out NAMENODE-specific test @Test public void testValidateConfigurations__hostGroupConfigForNameNodeHAPositive() throws Exception { Map> group2Props = new HashMap<>(); @@ -201,16 +205,18 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAPositive() t hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); BlueprintEntity entity = blueprint.toEntity(); + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.NONE); assertTrue(entity.getSecurityDescriptorReference() == null); } - @Test(expected= IllegalArgumentException.class) + @Ignore // move out NAMENODE-specific test + @Test(expected = IllegalArgumentException.class) public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHostGroups() throws Exception { Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); @@ -242,11 +248,13 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHos hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group3%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); + + // WHEN + new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); } - @Test(expected= IllegalArgumentException.class) + + @Ignore // move out NAMENODE-specific test + @Test(expected = IllegalArgumentException.class) public void testValidateConfigurations__hostGroupConfigForNameNodeHAMappedSameHostGroup() throws Exception { Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); @@ -278,76 +286,9 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAMappedSameHo hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - @Test(expected = InvalidTopologyException.class) - public void testValidateConfigurations__secretReference() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> group2Props = new HashMap<>(); - Map group2Category2Props = new HashMap<>(); - - group2Props.put("category2", group2Category2Props); - group2Category2Props.put("prop2", "val"); - hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test(expected = GPLLicenseNotAcceptedException.class) - public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", ImmutableMap.of(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS)); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - setupConfigurationWithGPLLicense(false); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test(expected = GPLLicenseNotAcceptedException.class) - public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", ImmutableMap.of(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS)); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - setupConfigurationWithGPLLicense(false); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test - public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", ImmutableMap.of( - BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS, - BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS)); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - private void setupConfigurationWithGPLLicense(boolean isGPLAllowed) { - reset(serverConfig); - expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce(); + // WHEN + new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index c4ba8d8ca6c..ac96d1912af 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -236,7 +236,7 @@ private Capture> testProcessWithKerberos(String blueprintP expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); @@ -250,7 +250,7 @@ private Capture> testProcessWithKerberos(String blueprintP services.add("HDFS"); services.add("KERBEROS"); services.add("ZOOKEPER"); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); expect(stack.getConfiguration(services)).andReturn(stackDefaultConfig).once(); List hdfsComponents = new ArrayList<>(); @@ -260,14 +260,14 @@ private Capture> testProcessWithKerberos(String blueprintP List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(topology.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(topology.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(topology.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.isValidConfigType("testConfigType")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("testConfigType")).andReturn(true).anyTimes(); expect(topology.getConfiguration()).andReturn(blueprintConfig).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); @@ -329,7 +329,7 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); @@ -341,7 +341,7 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex services.add("HDFS"); services.add("KERBEROS"); services.add("ZOOKEPER"); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); List hdfsComponents = new ArrayList<>(); hdfsComponents.add("NAMENODE"); @@ -350,9 +350,9 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(topology.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(topology.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(topology.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); @@ -396,14 +396,14 @@ public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() throws Exce expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); - expect(blueprint.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("admin-properties")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("yarn-site")).andReturn(false).anyTimes(); - expect(blueprint.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("global")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("admin-properties")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("yarn-site")).andReturn(false).anyTimes(); + expect(topology.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("global")).andReturn(true).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); @@ -446,12 +446,12 @@ public void testProcessClusterConfigRequestWithOnlyHostGroupConfigRemoveUnusedCo expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); - expect(blueprint.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("global")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("global")).andReturn(true).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 15dccb6b875..0ecfc5c4aa6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -17,6 +17,7 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; @@ -92,6 +93,7 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -99,9 +101,15 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; + @Mock(type = MockType.NICE) + private ClusterTopology topology; + @Mock(type = MockType.NICE) private Stack stack; @@ -188,8 +196,10 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -198,7 +208,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private List topologyValidators = new ArrayList<>(); - private Capture clusterTopologyCapture; private Capture> configRequestPropertiesCapture; private Capture> configRequestPropertiesCapture2; private Capture> configRequestPropertiesCapture3; @@ -207,7 +216,10 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Before public void setup() throws Exception { - clusterTopologyCapture = newCapture(); + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); configRequestPropertiesCapture3 = newCapture(); @@ -245,21 +257,22 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(topology.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); + expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); // don't expect toEntity() @@ -290,8 +303,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -304,29 +322,21 @@ public void setup() throws Exception { expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index 9a7f6a2e855..779babfa598 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -93,6 +94,7 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -100,9 +102,15 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; + @Mock(type = MockType.NICE) + private ClusterTopology topology; + @Mock(type = MockType.NICE) private Stack stack; @@ -184,8 +192,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -194,7 +204,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private List topologyValidators = new ArrayList<>(); - private Capture clusterTopologyCapture; private Capture> configRequestPropertiesCapture; private Capture> configRequestPropertiesCapture2; private Capture> configRequestPropertiesCapture3; @@ -204,7 +213,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Before public void setup() throws Exception { - clusterTopologyCapture = newCapture(); + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); configRequestPropertiesCapture3 = newCapture(); @@ -242,21 +254,22 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes(); + expect(topology.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); + expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); // don't expect toEntity() @@ -287,8 +300,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -301,35 +319,27 @@ public void setup() throws Exception { expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Arrays.asList("component1")) .anyTimes(); expect(group1.getComponentNames(ProvisionAction.START_ONLY)).andReturn(Collections.emptyList()) .anyTimes(); - expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Collections.emptyList()).anyTimes(); expect(group2.getComponentNames(ProvisionAction.START_ONLY)).andReturn(Collections.emptyList()) .anyTimes(); - expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index a91d953e9e4..d4fd957f264 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -94,6 +95,7 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -101,9 +103,15 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; + @Mock(type = MockType.NICE) + private ClusterTopology topology; + @Mock(type = MockType.NICE) private Stack stack; @@ -186,8 +194,10 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -205,6 +215,10 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Before public void setup() throws Exception { + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + clusterTopologyCapture = newCapture(); configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); @@ -243,21 +257,22 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(topology.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); + expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); // don't expect toEntity() @@ -288,8 +303,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -302,30 +322,22 @@ public void setup() throws Exception { expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java index e51ce5f63e1..8e866d8cf16 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java @@ -49,6 +49,7 @@ public class ClusterTopologyImplTest { private static final HostGroup group2 = createNiceMock(HostGroup.class); private static final HostGroup group3 = createNiceMock(HostGroup.class); private static final HostGroup group4 = createNiceMock(HostGroup.class); + private final AmbariContext ambariContext = createNiceMock(AmbariContext.class); private final Map hostGroupInfoMap = new HashMap<>(); private final Map hostGroupMap = new HashMap<>(); @@ -163,7 +164,7 @@ public void testCreate_duplicateHosts() throws Exception { replayAll(); // should throw exception due to duplicate host - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } @Test @@ -172,7 +173,7 @@ public void test_GetHostAssigmentForComponents() throws Exception { replayAll(); - new ClusterTopologyImpl(null, request).getHostAssignmentsForComponent("component1"); + new ClusterTopologyImpl(ambariContext, request).getHostAssignmentsForComponent("component1"); } @Test(expected = InvalidTopologyException.class) @@ -182,7 +183,7 @@ public void testCreate_NNHAInvaid() throws Exception { hostGroupInfoMap.get("group4").removeHost("host5"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); hostGroupInfoMap.get("group4").addHost("host5"); } @@ -194,7 +195,7 @@ public void testCreate_NNHAHostNameNotCorrectForStandby() throws Exception { bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "val"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } @Test(expected = IllegalArgumentException.class) @@ -205,7 +206,7 @@ public void testCreate_NNHAHostNameNotCorrectForActive() throws Exception { bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "host5"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } @Test(expected = IllegalArgumentException.class) @@ -216,7 +217,7 @@ public void testCreate_NNHAHostNameNotCorrectForStandbyWithActiveAsVariable() th bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "host6"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } private class TestTopologyRequest implements TopologyRequest { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java index a5265f67883..0bb725320a0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java @@ -97,6 +97,9 @@ public class LogicalRequestTest extends EasyMockSupport { @Mock private HostGroup hostGroup2; + @Mock + private Setting setting; + @Before public void setup() throws Exception { @@ -114,7 +117,8 @@ public void setup() throws Exception { expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_ONLY).anyTimes(); expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(blueprint.getName()).andReturn("blueprintDef").anyTimes(); - expect(blueprint.shouldSkipFailure()).andReturn(true).anyTimes(); + expect(blueprint.getSetting()).andReturn(setting).anyTimes(); + expect(setting.shouldSkipFailure()).andReturn(true).anyTimes(); PowerMock.reset(AmbariServer.class); @@ -122,9 +126,6 @@ public void setup() throws Exception { expect(AmbariServer.getController()).andReturn(controller).anyTimes(); PowerMock.replay(AmbariServer.class); - - - } @Test @@ -198,7 +199,7 @@ public void testPersistedRequestsWithReservedHosts() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); replayAll(); @@ -301,7 +302,7 @@ public void testPersistedRequestsWithHostPredicate() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_2"))).andReturn(hostGroup2).atLeastOnce(); - expect(hostGroup2.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent(hostGroup2)).andReturn(false).atLeastOnce(); replayAll(); @@ -412,7 +413,7 @@ public void testRemoveHostRequestByHostName() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); replayAll(); @@ -510,7 +511,7 @@ public void testRemovePendingHostRequests() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); replayAll(); @@ -596,7 +597,7 @@ private LogicalRequest createTopologyRequestByHostCount(int hostCount, String ho expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).anyTimes(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(hostRequests).anyTimes(); expect(blueprint.getHostGroup(eq(hostGroupEntity.getName()))).andReturn(hostGroup1).anyTimes(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).anyTimes(); + expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).anyTimes(); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 61a45c8b411..b5e09434e3c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -18,8 +18,10 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; @@ -111,6 +113,9 @@ public class TopologyManagerTest { @Rule public EasyMockRule mocks = new EasyMockRule(this); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @TestSubject private TopologyManager topologyManager = new TopologyManager(); @@ -192,8 +197,10 @@ public class TopologyManagerTest { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); - private Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -211,6 +218,11 @@ public class TopologyManagerTest { @Before public void setup() throws Exception { + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + replay(configuration); + clusterTopologyCapture = newCapture(); configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); @@ -249,19 +261,18 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(clusterTopologyMock.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(clusterTopologyMock.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopologyMock.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); + expect(clusterTopologyMock.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); // don't expect toEntity() @@ -280,12 +291,17 @@ public void setup() throws Exception { expect(stack.getComponents("service1")).andReturn(components1).anyTimes(); expect(stack.getComponents("service2")).andReturn(components2).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -297,27 +313,19 @@ public void setup() throws Exception { expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent(group1)).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent(group2)).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); expect(logicalRequestFactory.createRequest(eq(1L), (TopologyRequest) anyObject(), capture(clusterTopologyCapture))). diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java index c2fea1d43e0..ae144266885 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java @@ -63,7 +63,7 @@ public void before() { EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(clusterConfigurationMock).anyTimes(); EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + EasyMock.expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); } @After @@ -94,7 +94,7 @@ public void testShouldValidationPassWhenAllConfigTypesAreValid() throws Exceptio clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -117,7 +117,7 @@ public void testShouldValidationFailWhenInvalidConfigGroupsSpecifiedInCCTemplate clusterRequestConfigTypes = new HashSet<>(Arrays.asList("oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -138,7 +138,7 @@ public void testShouldValidationFailWhenThereIsAnInvalidConfigGroupProvided() th clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site", "oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java similarity index 87% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java index 46396877906..5d796bdcd73 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; import static java.util.stream.Collectors.toList; import static org.easymock.EasyMock.expect; @@ -39,6 +39,13 @@ import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; import org.easymock.EasyMock; import org.easymock.EasyMockRule; import org.easymock.Mock; @@ -50,15 +57,15 @@ import com.google.common.collect.ImmutableSet; -/** - * BlueprintValidatorImpl unit tests. - */ -public class BlueprintValidatorImplTest { +public class DependencyAndCardinalityValidatorTest { private final Map hostGroups = new LinkedHashMap<>(); @Rule public EasyMockRule mocks = new EasyMockRule(this); + @Mock(type = MockType.NICE) + private ClusterTopology topology; + @Mock(type = MockType.NICE) private Blueprint blueprint; @@ -102,10 +109,12 @@ public void setup() { autoDeploy.setEnabled(true); autoDeploy.setCoLocate("service1/component2"); + expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(new StackId("HDP", "2.2"))).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group1.getComponents()). @@ -148,9 +157,10 @@ public void testValidateTopology_basic() throws Exception { expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); } @Test(expected = InvalidTopologyException.class) @@ -164,9 +174,10 @@ public void testValidateTopology_basic_negative() throws Exception { expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.emptyList()).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); } @Test @@ -182,9 +193,9 @@ public void testValidateTopology_autoDeploy() throws Exception { expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); verify(group1); } @@ -219,10 +230,10 @@ public void testValidateTopology_autoDeploy_hasDependency() throws Exception { expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); expect(group1.addComponent(new Component("component3"))).andReturn(true).once(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); verify(group1); } @@ -239,7 +250,6 @@ public void testShouldDependencyBeExcludedWenRelatedServiceIsNotInBlueprint() th expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Cardinality cardinality = new Cardinality("1"); @@ -263,11 +273,11 @@ public void testShouldDependencyBeExcludedWenRelatedServiceIsNotInBlueprint() th expect(dependencyComponentInfo.isClient()).andReturn(true).anyTimes(); expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); @@ -285,7 +295,6 @@ public void testShouldThrowErrorWhenDependentComponentIsNotInBlueprint() throws expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Cardinality cardinality = new Cardinality("1"); @@ -306,11 +315,11 @@ public void testShouldThrowErrorWhenDependentComponentIsNotInBlueprint() throws expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); @@ -329,7 +338,6 @@ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIs expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Map> properties = new HashMap<>(); Map typeProps = new HashMap<>(); typeProps.put("yarn.resourcemanager.hostname", "testhost"); @@ -368,11 +376,11 @@ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIs expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes(); expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2); + replay(blueprint, topology, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java new file mode 100644 index 00000000000..2cb1ef10ed4 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.CODEC_CLASSES_PROPERTY_NAME; +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.LZO_CODEC_CLASS; +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.LZO_CODEC_CLASS_PROPERTY_NAME; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +public class GplPropertiesValidatorTest { + + private static final boolean ACCEPTED = true; + + private final org.apache.ambari.server.configuration.Configuration serverConfig = createNiceMock(org.apache.ambari.server.configuration.Configuration.class); + private final TopologyValidator validator = new GplPropertiesValidator(serverConfig); + + @Before + public void setup() { + reset(serverConfig); + } + + @After + public void tearDown() { + verify(serverConfig); + } + + @Test(expected = GPLLicenseNotAcceptedException.class) // THEN + public void rejectsCodecsPropertyWithLzoClassIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + } + + @Test(expected = GPLLicenseNotAcceptedException.class) // THEN + public void rejectsLzoCodecPropertyIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + LZO_CODEC_CLASS_PROPERTY_NAME, LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + } + + @Test + public void allowsLzoCodecIfGplIsAccepted() throws Exception { + // GIVEN + gpl(ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + LZO_CODEC_CLASS_PROPERTY_NAME, LZO_CODEC_CLASS, + CODEC_CLASSES_PROPERTY_NAME, "OtherCodec," + LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + @Test + public void allowsConfigWithoutReferenceToGplEvenIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + "fs.defaultFS", "hdfs://localhost:8020", + "io.compression.codecs", "org.apache.hadoop.io.compress.DefaultCodec" + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + private void gpl(boolean accepted) { + expect(serverConfig.getGplLicenseAccepted()).andReturn(accepted).atLeastOnce(); + replay(serverConfig); + } +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index 3ce0a024813..ee893185b09 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -14,15 +14,14 @@ package org.apache.ambari.server.topology.validators; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.easymock.EasyMock; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -32,27 +31,25 @@ import org.junit.Rule; import org.junit.Test; +import com.google.common.collect.ImmutableSet; + public class HiveServiceValidatorTest extends EasyMockSupport { @Rule public EasyMockRule mocks = new EasyMockRule(this); @Mock - private ClusterTopology clusterTopologyMock; - - @Mock - private Blueprint blueprintMock; + private ClusterTopology topology; @Mock - private Configuration configurationMock; + private Configuration configuration; @TestSubject - private HiveServiceValidator hiveServiceValidator = new HiveServiceValidator(); + private final TopologyValidator hiveServiceValidator = new HiveServiceValidator(); @Before public void setUp() throws Exception { - - + expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); } @After @@ -61,101 +58,88 @@ public void tearDown() throws Exception { } @Test - public void testShouldValidationPassWhenHiveServiceIsNotInBlueprint() throws Exception { - + public void allowsTopologyWithoutHive() throws Exception { // GIVEN - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(Collections.emptySet()); - replayAll(); + noHiveInTopology(); + noHiveConfig(); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenHiveServiceIsMissingConfigType() throws Exception { - + @Test + public void allowsExistingDatabaseWithoutMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "OOZIE"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(Collections.emptySet()); - - replayAll(); + topologyHasMysql(false); + hiveDatabaseIs("Existing MySQL"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } @Test - public void testShouldValidationPassWhenCustomHiveDatabaseSettingsProvided() throws Exception { - + public void allowsNewMysqlDatabaseWithMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "OOZIE"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("PSQL"); - replayAll(); + topologyHasMysql(true); + hiveDatabaseIs("New MySQL Database"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenDefaultsAreUsedAndMysqlComponentIsMissing() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsHiveWithoutConfig() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "HDFS"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(Collections.emptyList()).anyTimes(); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database"); - replayAll(); - + topologyHasMysql(true); + noHiveConfig(); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); - - // THEN - + hiveServiceValidator.validate(topology); } - @Test - public void testShouldValidationPassWhenDefaultsAreUsedAndMsqlComponentIsListed() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsNewMysqlDatabaseWithoutMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "HDFS", "MYSQL_SERVER"); - Collection hiveComponents = Arrays.asList("MYSQL_SERVER"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(hiveComponents).anyTimes(); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database"); - replayAll(); + topologyHasMysql(false); + hiveDatabaseIs("New MySQL Database"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); + } - // THEN + private void noHiveConfig() { + expect(configuration.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "hadoop-env")).anyTimes(); + replay(configuration); + } + + private void hiveDatabaseIs(String database) { + expect(configuration.getAllConfigTypes()).andReturn(Arrays.asList("hive-env", "hive-site", "core-site", "hadoop-env")); + expect(configuration.getPropertyValue("hive-env", "hive_database")).andReturn(database).anyTimes(); + replay(configuration); + } + private void topologyHasMysql(boolean hasMysql) { + ImmutableSet.Builder components = ImmutableSet.builder().add("HIVE_CLIENT", "HIVE_METASTORE", "HIVE_SERVER"); + if (hasMysql) { + components.add("MYSQL_SERVER"); + } + expect(topology.getComponentNames("HIVE")).andReturn(components.build()).anyTimes(); + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); + replay(topology); } + + private void noHiveInTopology() { + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN")).anyTimes(); + replay(topology); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java new file mode 100644 index 00000000000..484ade0bf6d --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; + +import java.util.stream.Stream; + +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class RejectUnknownComponentsTest { + + private TopologyValidator validator = new RejectUnknownComponents(); + private final ClusterTopology topology = createNiceMock(ClusterTopology.class); + private final StackDefinition stack = createNiceMock(StackDefinition.class); + + @Before + public void setUp() { + expect(topology.getStack()).andReturn(stack).anyTimes(); + } + + @After + public void tearDown() { + reset(topology, stack); + } + + @Test + public void acceptsKnownComponents() throws Exception { + // GIVEN + componentsInTopologyAre("VALID_COMPONENT", "ANOTHER_COMPONENT"); + validComponentsAre("VALID_COMPONENT", "ANOTHER_COMPONENT", "ONE_MORE_COMPONENT"); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + @Test(expected = InvalidTopologyException.class) + public void rejectsUnknownComponents() throws Exception { + // GIVEN + componentsInTopologyAre("VALID_COMPONENT", "UNKNOWN_COMPONENT"); + validComponentsAre("VALID_COMPONENT", "ANOTHER_COMPONENT"); + + // WHEN + validator.validate(topology); + } + + private void componentsInTopologyAre(String... components) { + expect(topology.getComponentNames()).andReturn(Stream.of(components)).anyTimes(); + replay(topology); + } + + private void validComponentsAre(String... components) { + expect(stack.getComponents()).andReturn(ImmutableSet.builder().add(components).build()).anyTimes(); + replay(stack); + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java index 780ca53bdd9..e249feb8da6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java @@ -99,17 +99,15 @@ public void setup() { EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); EasyMock.expect(blueprintMock.getHostGroups()).andReturn(hostGroups); - EasyMock.expect(blueprintMock.getServices()).andReturn(bpServices); - EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(bpServices); + EasyMock.expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); EasyMock.expect(masterHostGroupMock.getName()).andReturn("master").anyTimes(); EasyMock.expect(masterHostGroupMock.getConfiguration()).andReturn(masterHostGroupConfigurationMock).anyTimes(); - EasyMock.expect(masterHostGroupMock.getServices()).andReturn(masterHostGroupServices); EasyMock.expect(slaveHostGroupMock.getName()).andReturn("slave").anyTimes(); EasyMock.expect(slaveHostGroupMock.getConfiguration()).andReturn(slaveHostGroupConfigurationMock).anyTimes(); - EasyMock.expect(slaveHostGroupMock.getServices()).andReturn(slaveHostGroupServices); // there are 2 hostgroups to be considered by the test hostGroups.put("master", masterHostGroupMock); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java similarity index 94% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java index 722f5f1e31c..1f28e936680 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java @@ -16,12 +16,11 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; import static junit.framework.Assert.assertEquals; import static org.easymock.EasyMock.expect; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -30,7 +29,12 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.validators.RequiredPasswordValidator; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.HostGroupInfo; +import org.apache.ambari.server.topology.InvalidTopologyException; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -145,16 +149,10 @@ public void setup() { expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(Arrays.asList("component3")).anyTimes(); - expect(group1.getComponentNames("service3")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(Arrays.asList("component1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponentNames("service3")).andReturn(Arrays.asList("component4")).anyTimes(); expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); expect(stack.getServiceForComponent("component2")).andReturn("service1").anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java new file mode 100644 index 00000000000..fa3021f047e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.apache.ambari.server.topology.validators.TopologyValidatorTests.topologyWithProperties; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +public class SecretReferenceValidatorTest { + + private final TopologyValidator validator = new SecretReferenceValidator(); + + @Test + public void acceptsTopologyWithoutSecretReferences() throws InvalidTopologyException { + ClusterTopology topology = topologyWithProperties(ImmutableMap.of( + "hdfs-site", ImmutableMap.of( + "password", "secret" + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exceptions expected + } + + @Test(expected = InvalidTopologyException.class) + public void rejectsTopologyWithSecretReferences() throws InvalidTopologyException { + ClusterTopology topology = topologyWithProperties(ImmutableMap.of( + "hdfs-site", ImmutableMap.of( + "password", "SECRET:hdfs-site:1:test" + ) + )); + + // WHEN + validator.validate(topology); + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java index 2292170fee7..a03290a2ff0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java @@ -62,7 +62,7 @@ public void before() { expect(clusterTopologyMock.getConfiguration()).andReturn(clusterConfigurationMock).anyTimes(); expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); } @After diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java new file mode 100644 index 00000000000..2ae84412de1 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Configuration; + +/** + * Helper for TopologyValidator tests. + */ +public class TopologyValidatorTests { + + /** + * Creates a mock ClusterTopology with the given properties. + */ + static ClusterTopology topologyWithProperties(Map> properties) { + Configuration topologyConfig = new Configuration(properties, new HashMap<>()); + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getConfiguration()).andReturn(topologyConfig).anyTimes(); + replay(topology); + return topology; + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java index 015dc36ceb8..926bbd335e2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java @@ -76,7 +76,7 @@ public void skipsValidatingIrrelevantProperty() throws Exception { public void setUp() throws Exception { expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(clusterTopology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(SERVICE, CONFIG_TYPE)).andReturn(stackConfigWithMetadata).anyTimes(); } From 9e538432f796a09b2aea248e0044c6e6cc1e4efe Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 12 Feb 2018 17:54:59 +0100 Subject: [PATCH 14/25] Set blueprint config's parent; simplify security config validation --- .../internal/ClusterResourceProvider.java | 6 --- .../internal/ProvisionClusterRequest.java | 11 ++-- .../ambari/server/topology/AmbariContext.java | 1 + .../ambari/server/topology/Blueprint.java | 3 ++ .../ambari/server/topology/BlueprintImpl.java | 11 ++-- .../server/topology/ClusterTopologyImpl.java | 13 +++-- .../server/topology/TopologyManager.java | 51 +++++++------------ .../ClusterDeployWithStartOnlyTest.java | 4 +- ...stallWithoutStartOnComponentLevelTest.java | 4 +- .../ClusterInstallWithoutStartTest.java | 4 +- .../topology/ClusterTopologyImplTest.java | 5 ++ .../server/topology/TopologyManagerTest.java | 3 +- 12 files changed, 57 insertions(+), 59 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java index 7a1f517d4e8..f03231bfac1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java @@ -538,12 +538,6 @@ private RequestStatusResponse processBlueprintCreate(Map propert throw new IllegalArgumentException("Invalid Cluster Creation Template: " + e, e); } - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.NONE && - createClusterRequest.getBlueprint().getSecurity() != null && createClusterRequest.getBlueprint().getSecurity() - .getType() == SecurityType.KERBEROS) { - throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); - } - try { return topologyManager.provisionCluster(createClusterRequest); } catch (InvalidTopologyException e) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index 1af9248d021..b01fffdb41d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -19,6 +19,7 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; +import static org.apache.ambari.server.topology.TopologyManager.KDC_ADMIN_CREDENTIAL; import java.util.Arrays; import java.util.Collection; @@ -29,6 +30,7 @@ import org.apache.ambari.server.api.predicate.InvalidQueryException; import org.apache.ambari.server.security.encryption.CredentialStoreType; import org.apache.ambari.server.stack.NoSuchStackException; +import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileBuilder; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileEvaluationException; @@ -197,16 +199,17 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura } this.securityConfiguration = securityConfiguration; + this.credentialsMap = parseCredentials(properties); + if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS && getCredentialsMap().get(KDC_ADMIN_CREDENTIAL) == null) { + throw new InvalidTopologyTemplateException(KDC_ADMIN_CREDENTIAL + " is missing from request."); + } - Configuration configuration = configurationFactory.getConfiguration( - (Collection>) properties.get(CONFIGURATIONS_PROPERTY)); + Configuration configuration = configurationFactory.getConfiguration((Collection>) properties.get(CONFIGURATIONS_PROPERTY)); configuration.setParentConfiguration(blueprint.getConfiguration()); setConfiguration(configuration); parseHostGroupInfo(properties); - this.credentialsMap = parseCredentials(properties); - this.configRecommendationStrategy = parseConfigRecommendationStrategy(properties); setProvisionAction(parseProvisionAction(properties)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 8143c9fcb48..5b4dd4a70c2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -453,6 +453,7 @@ public void createAmbariHostResources(long clusterId, String hostName, Collecti for (Component component : components) { String serviceName = component.getServiceInstance(); + // FIXME handle null serviceName String componentName = component.getName(); try { if (cluster.getService(serviceName) != null && !componentName.equals(RootComponent.AMBARI_SERVER.name())) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 727e42ecd8a..3762df0b642 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; +import javax.annotation.Nonnull; + import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.state.StackId; @@ -93,6 +95,7 @@ public interface Blueprint { */ Collection getHostGroupsForComponent(String component); + @Nonnull SecurityConfiguration getSecurity(); /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index fc636eb44f8..e2fab706899 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -75,7 +75,6 @@ public BlueprintImpl(BlueprintEntity entity, Set stackIds) throws NoSuc // create config first because it is set as a parent on all host-group configs configuration = processConfiguration(entity.getConfigurations()); hostGroups = parseBlueprintHostGroups(entity); - // configuration.setParentConfiguration(stack.getConfiguration(getServices())); // FIXME services not available at this time, need to set parent config somewhere else setting = new Setting(parseSetting(entity.getSettings())); repoSettings = processRepoSettings(); } @@ -93,23 +92,21 @@ public BlueprintImpl(String name, Collection groups, Set sta hostGroups.put(hostGroup.getName(), hostGroup); } this.configuration = configuration; - // if the parent isn't set, the stack configuration is set as the parent - // parent is set to non-null, but empty config when exporting blueprint to prevent exporting stack default config -// if (configuration.getParentConfiguration() == null) { -// configuration.setParentConfiguration(stack.getConfiguration(getServices())); // FIXME services not available at this time -// } this.setting = setting != null ? setting : new Setting(ImmutableMap.of()); repoSettings = processRepoSettings(); } + @Override public String getName() { return name; } + @Override public Set getStackIds() { return stackIds; } + @Override public SecurityConfiguration getSecurity() { return security; } @@ -151,6 +148,7 @@ public Collection getHostGroupsForComponent(String component) { return resultGroups; } + @Override public BlueprintEntity toEntity() { BlueprintEntity entity = new BlueprintEntity(); entity.setBlueprintName(name); @@ -438,6 +436,7 @@ private List processRepoSettings() { return setting != null ? setting.processRepoSettings() : Collections.emptyList(); } + @Override public List getRepositorySettings(){ return repoSettings; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index 79bb417dbc6..d2bab7c4a66 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; /** * Represents a cluster topology. @@ -75,17 +76,20 @@ public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topology if (topologyRequest instanceof ProvisionClusterRequest) { ProvisionClusterRequest provisionRequest = (ProvisionClusterRequest) topologyRequest; defaultPassword = provisionRequest.getDefaultPassword(); - stackIds = provisionRequest.getStackIds(); + stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), provisionRequest.getStackIds())); + stack = ambariContext.composeStacks(stackIds); + setBlueprintParentConfig(); } else { defaultPassword = null; stackIds = ImmutableSet.of(); + stack = null; } - stack = ambariContext.composeStacks(stackIds); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); + } - // todo extract validation to specialized service - validateTopology(); + void setBlueprintParentConfig() { + blueprint.getConfiguration().setParentConfiguration(stack.getConfiguration(getServices())); } @Override @@ -280,6 +284,7 @@ static boolean isYarnResourceManagerHAEnabled(Map> c && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true"); } + // FIXME move out private void validateTopology() throws InvalidTopologyException { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 25a56d405d9..89bacb74e9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -51,7 +51,6 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.RequestImpl; import org.apache.ambari.server.controller.internal.ScaleClusterRequest; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; @@ -278,31 +277,19 @@ public boolean isClusterProvisionWithBlueprintFinished(long clusterId) { public RequestStatusResponse provisionCluster(final ProvisionClusterRequest request) throws InvalidTopologyException, AmbariException { ensureInitialized(); - final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request); + final ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, request); final String clusterName = request.getClusterName(); - final StackDefinition stack = topology.getStack(); final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); // get the id prior to creating ambari resources which increments the counter final Long provisionId = ambariContext.getNextRequestId(); - SecurityType securityType = null; - Credential credential = null; - SecurityConfiguration securityConfiguration = processSecurityConfiguration(request); - - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS) { - securityType = SecurityType.KERBEROS; - addKerberosClient(topology); - + SecurityType securityType = securityConfiguration.getType(); + if (securityType == SecurityType.KERBEROS && addKerberosClient(topology)) { // refresh default stack config after adding KERBEROS_CLIENT component to topology - topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getServices())); - - credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); - if (credential == null) { - throw new InvalidTopologyException(KDC_ADMIN_CREDENTIAL + " is missing from request."); - } + topology.setBlueprintParentConfig(); } topologyValidatorService.validateTopologyConfiguration(topology); @@ -310,11 +297,12 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ // create resources ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID); - if (securityConfiguration != null && securityConfiguration.getDescriptor() != null) { + if (securityConfiguration.getDescriptor() != null) { submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor()); } - if (credential != null) { + if (securityType == SecurityType.KERBEROS) { + Credential credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); submitCredential(clusterName, credential); } @@ -425,31 +413,28 @@ private void submitCredential(String clusterName, Credential credential) { /** * Retrieve security info from Blueprint if missing from Cluster Template request. - * - * @param request - * @return */ private SecurityConfiguration processSecurityConfiguration(ProvisionClusterRequest request) { - LOG.debug("Getting security configuration from the request ..."); SecurityConfiguration securityConfiguration = request.getSecurityConfiguration(); + SecurityConfiguration blueprintSecurity = request.getBlueprint().getSecurity(); if (securityConfiguration == null) { - // todo - perform this logic at request creation instead! LOG.debug("There's no security configuration in the request, retrieving it from the associated blueprint"); - securityConfiguration = request.getBlueprint().getSecurity(); - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS && - securityConfiguration.getDescriptorReference() != null) { - securityConfiguration = securityConfigurationFactory.loadSecurityConfigurationByReference - (securityConfiguration.getDescriptorReference()); + securityConfiguration = blueprintSecurity; + if (securityConfiguration.getType() == SecurityType.KERBEROS && securityConfiguration.getDescriptorReference() != null) { + securityConfiguration = securityConfigurationFactory.loadSecurityConfigurationByReference(securityConfiguration.getDescriptorReference()); } + } else if (securityConfiguration.getType() == SecurityType.NONE && blueprintSecurity.getType() == SecurityType.KERBEROS) { + throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); } + return securityConfiguration; } private void submitKerberosDescriptorAsArtifact(String clusterName, String descriptor) { ResourceProvider artifactProvider = - ambariContext.getClusterController().ensureResourceProvider(Resource.Type.Artifact); + AmbariContext.getClusterController().ensureResourceProvider(Resource.Type.Artifact); Map properties = new HashMap<>(); properties.put(ArtifactResourceProvider.ARTIFACT_NAME_PROPERTY, "kerberos_descriptor"); @@ -1098,10 +1083,12 @@ private boolean isHostIgnored(String host) { * * @param topology cluster topology */ - private void addKerberosClient(ClusterTopology topology) { + private boolean addKerberosClient(ClusterTopology topology) { + boolean changed = false; for (HostGroup group : topology.getBlueprint().getHostGroups().values()) { - group.addComponent(new Component("KERBEROS_CLIENT")); + changed |= group.addComponent(new Component("KERBEROS_CLIENT")); } + return changed; } /** diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 0ecfc5c4aa6..618b00192d3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -27,7 +27,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import static org.junit.Assert.assertEquals; @@ -274,6 +273,7 @@ public void setup() throws Exception { expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -356,7 +356,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index 779babfa598..a15318eb4fa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -29,7 +29,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import java.lang.reflect.Field; @@ -271,6 +270,7 @@ public void setup() throws Exception { expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -359,7 +359,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), (String) eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index d4fd957f264..8e03c33a13c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -29,7 +29,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import java.lang.reflect.Field; @@ -274,6 +273,7 @@ public void setup() throws Exception { expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -357,7 +357,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java index 8e866d8cf16..5e81730e28d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java @@ -33,6 +33,7 @@ import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; /** @@ -176,6 +177,7 @@ public void test_GetHostAssigmentForComponents() throws Exception { new ClusterTopologyImpl(ambariContext, request).getHostAssignmentsForComponent("component1"); } + @Ignore @Test(expected = InvalidTopologyException.class) public void testCreate_NNHAInvaid() throws Exception { bpconfiguration.setProperty("hdfs-site", "dfs.nameservices", "val"); @@ -187,6 +189,7 @@ public void testCreate_NNHAInvaid() throws Exception { hostGroupInfoMap.get("group4").addHost("host5"); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForStandby() throws Exception { expect(group4.getName()).andReturn("group4"); @@ -198,6 +201,7 @@ public void testCreate_NNHAHostNameNotCorrectForStandby() throws Exception { new ClusterTopologyImpl(ambariContext, request); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForActive() throws Exception { expect(group4.getName()).andReturn("group4"); @@ -209,6 +213,7 @@ public void testCreate_NNHAHostNameNotCorrectForActive() throws Exception { new ClusterTopologyImpl(ambariContext, request); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForStandbyWithActiveAsVariable() throws Exception { expect(group4.getName()).andReturn("group4"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index b5e09434e3c..b50c3ca1604 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -275,6 +275,7 @@ public void setup() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -337,7 +338,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), (String) isNull(), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), isNull(), anyLong()); expectLastCall().anyTimes(); expect(ambariContext.getNextRequestId()).andReturn(1L).anyTimes(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); From a2cead9b1183d7e688370b197fd52699a080de48 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 Feb 2018 13:48:13 +0100 Subject: [PATCH 15/25] Resolve components --- .../controller/internal/CompositeStack.java | 8 +- .../server/controller/internal/Stack.java | 16 +- .../controller/internal/StackDefinition.java | 12 +- .../ambari/server/topology/AmbariContext.java | 3 +- .../server/topology/ClusterTopology.java | 15 ++ .../server/topology/ClusterTopologyImpl.java | 46 ++++++ .../server/topology/ComponentResolver.java | 151 ++++++++++++++++++ .../server/topology/ResolvedComponent.java | 52 ++++++ .../server/topology/TopologyManager.java | 10 +- 9 files changed, 291 insertions(+), 22 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java index b3316738476..333a20aebf8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java @@ -27,6 +27,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ConfigHelper; @@ -207,10 +209,10 @@ public String getServiceForComponent(String component) { } @Override - public Collection getServicesForComponents(Collection components) { + @Nonnull + public Stream> getServicesForComponent(String component) { return stacks.stream() - .flatMap(m -> m.getServicesForComponents(components).stream()) - .collect(toSet()); + .flatMap(stack -> stack.getServicesForComponent(component)); } @Override diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index 15038f195e0..af34c711985 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -26,6 +26,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.ReadOnlyConfigurationResponse; @@ -41,6 +43,7 @@ import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; +import org.apache.commons.lang3.tuple.Pair; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -370,13 +373,12 @@ public String getServiceForComponent(String component) { } @Override - public Collection getServicesForComponents(Collection components) { - Set services = new HashSet<>(); - for (String component : components) { - services.add(getServiceForComponent(component)); - } - - return services; + @Nonnull + public Stream> getServicesForComponent(String component) { + String service = getServiceForComponent(component); + return service != null + ? Stream.of(Pair.of(getStackId(), service)) + : Stream.empty(); } @Override diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java index 29041e681db..3f9c74f933a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java @@ -23,6 +23,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.DependencyInfo; @@ -31,6 +33,7 @@ import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.validators.DependencyAndCardinalityValidator; +import org.apache.commons.lang3.tuple.Pair; /** * Encapsulates stack information. @@ -201,13 +204,10 @@ public interface StackDefinition { String getServiceForComponent(String component); /** - * Get the names of the services which contains the specified components. - * - * @param components collection of components - * - * @return collection of services which contain the specified components + * Get (stackID, service) pairs which contain the specified component in this stack. */ - Collection getServicesForComponents(Collection components); + @Nonnull + Stream> getServicesForComponent(String component); /** * Obtain the service name which corresponds to the specified configuration. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 5b4dd4a70c2..761d59efbcc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -452,9 +452,8 @@ public void createAmbariHostResources(long clusterId, String hostName, Collecti final Set requests = new HashSet<>(); for (Component component : components) { - String serviceName = component.getServiceInstance(); - // FIXME handle null serviceName String componentName = component.getName(); + String serviceName = component.getServiceInstance(); try { if (cluster.getService(serviceName) != null && !componentName.equals(RootComponent.AMBARI_SERVER.name())) { requests.add(new ServiceComponentHostRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, serviceName, componentName, hostName, null)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index a9ba9a975dd..9e31c648dad 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -122,6 +122,19 @@ public interface ClusterTopology { */ Collection getServices(); + /** + * @return the mpack instance defined in the topology with the given name + */ + MpackInstance getMpack(String name); + + Map> getServicesByMpack(); + + /** + * @return service instances defined in the topology, mapped by service name, + * whose name is unique across all mpacks. + */ + Map getUniqueServices(); + /** * Get all of the components represented in the blueprint. * @@ -257,4 +270,6 @@ public interface ClusterTopology { * @return true if the host group contains a master component; false otherwise */ boolean containsMasterComponent(HostGroup hostGroup); + + Iterable getHostGroups(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index d2bab7c4a66..59ecc021351 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; @@ -30,6 +31,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; @@ -43,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; @@ -65,6 +68,7 @@ public class ClusterTopologyImpl implements ClusterTopology { private final Map hostGroupInfoMap = new HashMap<>(); private final AmbariContext ambariContext; private final String defaultPassword; + private final Map mpacks; public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topologyRequest) throws InvalidTopologyException { this.ambariContext = ambariContext; @@ -73,16 +77,28 @@ public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topology this.blueprint = topologyRequest.getBlueprint(); this.configuration = topologyRequest.getConfiguration(); + // FIXME smelling more and more + // 1. shouldn't have to cast + // 2. what about replayed request + // -> + // 1. interface to combine blueprint and provision request + // 2. replayed request should simply be a provision or scale request + // 3. do not create a ClusterTopologyImpl for scale request -- create for original provision request only if (topologyRequest instanceof ProvisionClusterRequest) { ProvisionClusterRequest provisionRequest = (ProvisionClusterRequest) topologyRequest; defaultPassword = provisionRequest.getDefaultPassword(); stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), provisionRequest.getStackIds())); stack = ambariContext.composeStacks(stackIds); setBlueprintParentConfig(); + mpacks = ImmutableMap.copyOf(Stream.concat(blueprint.getMpacks().stream(), provisionRequest.getMpackInstances().stream()) + .collect(toMap( + MpackInstance::getMpackName, + Function.identity()))); } else { defaultPassword = null; stackIds = ImmutableSet.of(); stack = null; + mpacks = ImmutableMap.of(); } registerHostGroupInfo(topologyRequest.getHostGroupInfo()); @@ -132,6 +148,11 @@ public Map getHostGroupInfo() { return hostGroupInfoMap; } + @Override + public Iterable getHostGroups() { + return blueprint.getHostGroups().values(); + } + //todo: do we want to return groups with no requested hosts? @Override public Collection getHostGroupsForComponent(String component) { @@ -207,6 +228,31 @@ public Collection getServices() { .collect(toSet()); } + @Override + public MpackInstance getMpack(String name) { + return mpacks.get(name); + } + + @Override + public Map> getServicesByMpack() { + Map> result = new HashMap<>(); + for (MpackInstance mpack : mpacks.values()) { + Map services = mpack.getServiceInstances().stream() + .collect(toMap(ServiceInstance::getName, Function.identity())); + result.put(mpack.getMpackName(), services); + } + return result; + } + + @Override + public Map getUniqueServices() { + Map map = mpacks.values().stream() + .flatMap(mpack -> mpack.getServiceInstances().stream()) + .collect(toMap(ServiceInstance::getName, Function.identity(), (s1, s2) -> null)); + map.entrySet().removeIf(e -> e.getValue() == null); // remove non-unique names mapped to null + return map; + } + @Override public Stream getComponentNames() { return getComponents() diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java new file mode 100644 index 00000000000..25d0c2b3056 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static java.util.stream.Collectors.toSet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.StackId; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; + +/** + * Resolves all incompletely specified host group components in the topology: + * finds stack and/or service type that each component is defined in. + */ +public class ComponentResolver { + + private static final Logger LOG = LoggerFactory.getLogger(ComponentResolver.class); + + private final ClusterTopology topology; + private final Map uniqueServices; + private final Map> mpackServices; + + public ComponentResolver(ClusterTopology topology) { + this.topology = topology; + + uniqueServices = topology.getUniqueServices(); + mpackServices = topology.getServicesByMpack(); + } + + public Map> resolve() { + Map> result = new HashMap<>(); + List problems = new LinkedList<>(); + + StackDefinition stack = topology.getStack(); + for (HostGroup hg : topology.getHostGroups()) { + result.put(hg.getName(), new HashSet<>()); + + for (Component comp : hg.getComponents()) { + Stream> servicesForComponent = stack.getServicesForComponent(comp.getName()); + servicesForComponent = filterByMpackName(comp, servicesForComponent); + servicesForComponent = filterByServiceName(comp, servicesForComponent); + + Set> serviceMatches = servicesForComponent.collect(toSet()); + + if (serviceMatches.size() != 1) { + String msg = formatResolutionProblemMessage(hg, comp, serviceMatches); + LOG.warn("Component resolution failure:" + msg); + problems.add(msg); + } else { + Pair stackService = serviceMatches.iterator().next(); + StackId stackId = stackService.getLeft(); + String serviceType = stackService.getRight(); + String serviceName = comp.getServiceInstance(); + if (Strings.isNullOrEmpty(serviceName)) { + serviceName = serviceType; + } + + ResolvedComponent resolved = new ResolvedComponent(stackId, serviceName, serviceType, comp); + LOG.debug("Component resolved: " + resolved); + result.get(hg.getName()).add(resolved); + } + } + } + + if (!problems.isEmpty()) { + throw new IllegalArgumentException("Component resolution failure:\n" + Joiner.on("\n").join(problems)); + } + + return result; + } + + private static String formatResolutionProblemMessage(HostGroup hg, Component comp, Set> serviceMatches) { + boolean multipleMatches = !serviceMatches.isEmpty(); + String problem = multipleMatches ? "Multiple services" : "No service"; + + StringBuilder sb = new StringBuilder(problem) + .append(" found for component ").append(comp.getName()) + .append(" in host group " ).append(hg.getName()); + + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + sb.append(" mpack: ").append(comp.getMpackInstance()); + } + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + sb.append(" service: ").append(comp.getServiceInstance()); + } + if (multipleMatches) { + sb.append(": ").append(serviceMatches); + } + + return sb.toString(); + } + + // if component references a specific service instance, filter the stream by the type of that service + private Stream> filterByServiceName(Component comp, Stream> stream) { + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + String mpackName = comp.getMpackInstance(); + Map services = !Strings.isNullOrEmpty(mpackName) + ? mpackServices.get(mpackName) + : uniqueServices; + + ServiceInstance service = services.get(comp.getServiceInstance()); + if (service != null) { + String serviceType = service.getType(); + + return stream + .filter(pair -> pair.getRight().equals(serviceType)); + } + } + + return stream; + } + + // if component references a specific mpack instance, filter the stream by the name of that mpack + private Stream> filterByMpackName(Component comp, Stream> stream) { + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + return stream + .filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); + } + + return stream; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java new file mode 100644 index 00000000000..259c27523ac --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.state.StackId; + +public class ResolvedComponent { + + private final StackId stackId; + private final String serviceName; + private final String serviceType; + private final Component component; + + public ResolvedComponent(StackId stackId, String serviceName, String serviceType, Component component) { + this.stackId = stackId; + this.serviceName = serviceName; + this.serviceType = serviceType; + this.component = component; + } + + public StackId getStackId() { + return stackId; + } + + public String getServiceName() { + return serviceName; + } + + public String getServiceType() { + return serviceType; + } + + public Component getComponent() { + return component; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 89bacb74e9c..39ea3bbdc34 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -282,9 +282,6 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); - // get the id prior to creating ambari resources which increments the counter - final Long provisionId = ambariContext.getNextRequestId(); - SecurityConfiguration securityConfiguration = processSecurityConfiguration(request); SecurityType securityType = securityConfiguration.getType(); if (securityType == SecurityType.KERBEROS && addKerberosClient(topology)) { @@ -292,7 +289,12 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ topology.setBlueprintParentConfig(); } - topologyValidatorService.validateTopologyConfiguration(topology); + Map> resolved = new ComponentResolver(topology).resolve(); + // FIXME use resolved components in topology + topologyValidatorService.validateTopologyConfiguration(topology); // FIXME known stacks validation is too late here + + // get the id prior to creating ambari resources which increments the counter + final Long provisionId = ambariContext.getNextRequestId(); // create resources ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID); From 1568c1762abf807cd3193a6ba7fd06d616198030 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 13 Feb 2018 20:40:13 +0100 Subject: [PATCH 16/25] Combine Blueprint and ProvisionClusterRequest --- .../render/ClusterBlueprintRenderer.java | 5 +- .../StackAdvisorBlueprintProcessor.java | 2 +- .../controller/ServiceComponentRequest.java | 27 ++ .../server/controller/ServiceRequest.java | 30 ++ .../BlueprintConfigurationProcessor.java | 62 +-- .../internal/ProvisionClusterRequest.java | 9 +- .../ambari/server/topology/AmbariContext.java | 65 ++- .../ambari/server/topology/Blueprint.java | 6 +- ...BlueprintBasedClusterProvisionRequest.java | 213 ++++++++++ .../ambari/server/topology/BlueprintImpl.java | 12 +- .../topology/ClusterConfigurationRequest.java | 47 +-- .../server/topology/ClusterTopology.java | 100 ++--- .../server/topology/ClusterTopologyImpl.java | 195 +++------ .../server/topology/ComponentResolver.java | 21 +- .../ambari/server/topology/HostRequest.java | 6 +- .../server/topology/LogicalRequest.java | 8 +- .../server/topology/PersistedStateImpl.java | 10 +- .../server/topology/ProvisionRequest.java | 17 + .../server/topology/ResolvedComponent.java | 11 + .../server/topology/TopologyManager.java | 73 +--- .../server/topology/TopologyRequest.java | 4 - .../DependencyAndCardinalityValidator.java | 59 +-- .../validators/HiveServiceValidator.java | 3 +- .../NameNodeHighAvailabilityValidator.java | 15 +- .../validators/RejectUnknownComponents.java | 4 +- .../validators/RequiredPasswordValidator.java | 110 ++--- .../render/ClusterBlueprintRendererTest.java | 2 - .../AmbariManagementControllerTest.java | 6 +- .../BlueprintConfigurationProcessorTest.java | 389 +++++++++--------- .../server/topology/AmbariContextTest.java | 8 +- .../ClusterConfigurationRequestTest.java | 16 +- .../ClusterDeployWithStartOnlyTest.java | 11 +- ...stallWithoutStartOnComponentLevelTest.java | 11 +- .../ClusterInstallWithoutStartTest.java | 11 +- ...tsTest.java => ComponentResolverTest.java} | 12 +- .../server/topology/LogicalRequestTest.java | 10 +- .../server/topology/TopologyManagerTest.java | 11 +- .../validators/HiveServiceValidatorTest.java | 3 +- 38 files changed, 825 insertions(+), 779 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java rename ambari-server/src/test/java/org/apache/ambari/server/topology/{validators/RejectUnknownComponentsTest.java => ComponentResolverTest.java} (86%) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java index deb13db607a..e518de7b68f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java @@ -197,7 +197,7 @@ private Resource createBlueprintResource(TreeNode clusterNode) { BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); - Set stackIds = topology.getBlueprint().getStackIds(); + Set stackIds = topology.getStackIds(); if (stackIds.size() == 1) { StackId stackId = Iterables.getOnlyElement(stackIds); blueprintResource.setProperty("Blueprints/stack_name", stackId.getStackName()); @@ -210,8 +210,7 @@ private Resource createBlueprintResource(TreeNode clusterNode) { try { String clusterName = topology.getAmbariContext().getClusterName(topology.getClusterId()); - Map kerberosDescriptor = getKerberosDescriptor(topology.getAmbariContext() - .getClusterController(), clusterName); + Map kerberosDescriptor = getKerberosDescriptor(AmbariContext.getClusterController(), clusterName); if (kerberosDescriptor != null) { securityConfigMap.put(SecurityConfigurationFactory.KERBEROS_DESCRIPTOR_PROPERTY_ID, kerberosDescriptor); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java index b6f8d667e6e..87b64a56e81 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java @@ -75,7 +75,7 @@ public static void init(StackAdvisorHelper instance) { * @param userProvidedConfigurations User configurations of cluster provided in Blueprint + Cluster template */ public void adviseConfiguration(ClusterTopology clusterTopology, Map> userProvidedConfigurations) throws ConfigurationTopologyException { - for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) { + for (StackId stackId : clusterTopology.getStackIds()) { StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, stackId, StackAdvisorRequestType.CONFIGURATIONS); try { RecommendationResponse response = stackAdvisorHelper.recommend(request); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java index f59eb984e7f..0c73933c753 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java @@ -19,6 +19,8 @@ package org.apache.ambari.server.controller; +import java.util.Objects; + public class ServiceComponentRequest { private String clusterName; // REF @@ -145,4 +147,29 @@ public String toString() { "desiredState=%s, recoveryEnabled=%s, componentCategory=%s]", clusterName, serviceGroupName, serviceName, clusterName, desiredState, recoveryEnabled, componentCategory); } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ServiceComponentRequest other = (ServiceComponentRequest) obj; + + return Objects.equals(clusterName, other.clusterName) && + Objects.equals(serviceGroupName, other.serviceGroupName) && + Objects.equals(serviceName, other.serviceName) && + Objects.equals(componentCategory, other.componentCategory) && + Objects.equals(componentName, other.componentName) && + Objects.equals(desiredState, other.desiredState) && + Objects.equals(recoveryEnabled, other.recoveryEnabled); + } + + @Override + public int hashCode() { + return Objects.hash(clusterName, serviceGroupName, serviceName, componentCategory, componentName, desiredState, recoveryEnabled); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java index 70e5240c38a..4516b555069 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java @@ -17,6 +17,8 @@ */ package org.apache.ambari.server.controller; +import java.util.Objects; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.StackId; @@ -196,4 +198,32 @@ public String toString() { public void setResolvedRepository(RepositoryVersionEntity repositoryVersion) { resolvedRepository = repositoryVersion; } public RepositoryVersionEntity getResolvedRepository() { return resolvedRepository; } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ServiceRequest other = (ServiceRequest) obj; + + return Objects.equals(clusterName, other.clusterName) && + Objects.equals(serviceGroupName, other.serviceGroupName) && + Objects.equals(serviceType, other.serviceType) && + Objects.equals(serviceName, other.serviceName) && + Objects.equals(desiredState, other.desiredState) && + Objects.equals(maintenanceState, other.maintenanceState) && + Objects.equals(credentialStoreEnabled, other.credentialStoreEnabled) && + Objects.equals(credentialStoreSupported, other.credentialStoreSupported) && + Objects.equals(desiredStackId, other.desiredStackId) && + Objects.equals(desiredRepositoryVersionId, other.desiredRepositoryVersionId); + } + + @Override + public int hashCode() { + return Objects.hash(clusterName, serviceGroupName, serviceType, serviceName, desiredState, maintenanceState, credentialStoreEnabled, credentialStoreSupported, desiredStackId, desiredRepositoryVersionId); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 3687ef90908..e3efb36568c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -44,7 +44,6 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.AdvisedConfiguration; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; @@ -169,6 +168,23 @@ public class BlueprintConfigurationProcessor { private static Set configPropertiesWithHASupport = new HashSet<>(Arrays.asList("fs.defaultFS", "hbase.rootdir", "instance.volumes", "policymgr_external_url", "xasecure.audit.destination.hdfs.dir")); + public static boolean isNameNodeHAEnabled(Map> configurationProperties) { + return configurationProperties.containsKey("hdfs-site") && + (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") || + configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices")); + } + + /** + * Static convenience function to determine if Yarn ResourceManager HA is enabled + * @param configProperties configuration properties for this cluster + * @return true if Yarn ResourceManager HA is enabled + * false if Yarn ResourceManager HA is not enabled + */ + public static boolean isYarnResourceManagerHAEnabled(Map> configProperties) { + return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled") + && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true"); + } + /** * Statically-defined list of filters to apply on property exports. * This will initially be used to filter out the Ranger Passwords, but @@ -354,7 +370,8 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept // set of properties (copy) doesn't include the removed properties. If an updater // removes a property other than the property it is registered for then we will // have an issue as it won't be removed from the clusterProps map as it is a copy. - Map> clusterProps = clusterConfig.getFullProperties(); + Map> configProperties = clusterConfig.getFullProperties(); + Map> clusterProps = configProperties; for (Map> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry> entry : updaterMap.entrySet()) { String type = entry.getKey(); @@ -402,10 +419,10 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept } //todo: lots of hard coded HA rules included here - if (clusterTopology.isNameNodeHAEnabled()) { + if (isNameNodeHAEnabled(configProperties)) { // add "dfs.internal.nameservices" if it's not specified - Map hdfsSiteConfig = clusterConfig.getFullProperties().get("hdfs-site"); + Map hdfsSiteConfig = configProperties.get("hdfs-site"); String nameservices = hdfsSiteConfig.get("dfs.nameservices"); String int_nameservices = hdfsSiteConfig.get("dfs.internal.nameservices"); if(int_nameservices == null && nameservices != null) { @@ -490,15 +507,16 @@ private static boolean shouldPropertyBeStoredWithDefault(String propertyName) { */ public void doUpdateForBlueprintExport() { // HA configs are only processed in cluster configuration, not HG configurations - if (clusterTopology.isNameNodeHAEnabled()) { + Map> configProperties = clusterTopology.getConfiguration().getFullProperties(); + if (isNameNodeHAEnabled(configProperties)) { doNameNodeHAUpdate(); } - if (clusterTopology.isYarnResourceManagerHAEnabled()) { + if (isYarnResourceManagerHAEnabled(configProperties)) { doYarnResourceManagerHAUpdate(); } - if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + if (isOozieServerHAEnabled(configProperties)) { doOozieServerHAUpdate(); } @@ -626,7 +644,6 @@ private void doRecommendConfigurations(Configuration configuration, Set * @param advisedConfigurations advised configuration instance */ private void doFilterStackDefaults(Map advisedConfigurations) { - Blueprint blueprint = clusterTopology.getBlueprint(); Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServices()); Map> stackDefaultProps = stackDefaults.getProperties(); for (Map.Entry adConfEntry : advisedConfigurations.entrySet()) { @@ -705,15 +722,16 @@ private void doRemovePropertiesIfNeeded(Configuration configuration, private Collection>> createCollectionOfUpdaters() { Collection>> updaters = allUpdaters; - if (clusterTopology.isNameNodeHAEnabled()) { + Map> configProperties = clusterTopology.getConfiguration().getFullProperties(); + if (isNameNodeHAEnabled(configProperties)) { updaters = addNameNodeHAUpdaters(updaters); } - if (clusterTopology.isYarnResourceManagerHAEnabled()) { + if (isYarnResourceManagerHAEnabled(configProperties)) { updaters = addYarnResourceManagerHAUpdaters(updaters); } - if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + if (isOozieServerHAEnabled(configProperties)) { updaters = addOozieServerHAUpdaters(updaters); } @@ -1473,7 +1491,8 @@ public String updateForClusterCreate(String propertyName, if (matchingGroupCount == 0 && cardinality.isValidCount(0)) { return origValue; } else { - if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroupCount == 2)) { + Map> configProperties = topology.getConfiguration().getFullProperties(); + if (isComponentNameNode() && (matchingGroupCount == 2) && isNameNodeHAEnabled(configProperties)) { // if this is the defaultFS property, it should reflect the nameservice name, // rather than a hostname (used in non-HA scenarios) if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) { @@ -1499,13 +1518,13 @@ public String updateForClusterCreate(String propertyName, } - if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) { + if (isComponentSecondaryNameNode() && (matchingGroupCount == 0) && isNameNodeHAEnabled(configProperties)) { // if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE // eventually this type of information should be encoded in the stacks return origValue; } - if (topology.isYarnResourceManagerHAEnabled() && isComponentResourceManager() && (matchingGroupCount == 2)) { + if (isComponentResourceManager() && (matchingGroupCount == 2) && isYarnResourceManagerHAEnabled(configProperties)) { if (!origValue.contains("localhost")) { // if this Yarn property is a FQDN, then simply return it return origValue; @@ -2067,7 +2086,7 @@ public Collection getRequiredHostGroups(String propertyName, while (m.find()) { String groupName = m.group(1); - if (!topology.getBlueprint().getHostGroups().containsKey(groupName)) { + if (!topology.getHostGroupInfo().containsKey(groupName)) { throw new IllegalArgumentException( "Unable to match blueprint host group token to a host group: " + groupName); } @@ -2990,7 +3009,7 @@ private void setStackToolsAndFeatures(Configuration configuration, Set c ); try { - for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) { + for (StackId stackId : clusterTopology.getStackIds()) { Map> defaultStackProperties = configHelper.getDefaultStackProperties(stackId); if (defaultStackProperties.containsKey(CLUSTER_ENV_CONFIG_TYPE_NAME)) { Map clusterEnvDefaultProperties = defaultStackProperties.get(CLUSTER_ENV_CONFIG_TYPE_NAME); @@ -3320,7 +3339,7 @@ private static class HDFSNameNodeHAFilter implements PropertyFilter { * namenode. */ private final Set setOfHDFSPropertyNamesNonHA = - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("dfs.namenode.http-address", "dfs.namenode.https-address", "dfs.namenode.rpc-address"))); + ImmutableSet.of("dfs.namenode.http-address", "dfs.namenode.https-address", "dfs.namenode.rpc-address"); /** @@ -3335,13 +3354,8 @@ private static class HDFSNameNodeHAFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - if (topology.isNameNodeHAEnabled()) { - if (setOfHDFSPropertyNamesNonHA.contains(propertyName)) { - return false; - } - } - - return true; + return !setOfHDFSPropertyNamesNonHA.contains(propertyName) + || !isNameNodeHAEnabled(topology.getConfiguration().getFullProperties()); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index b01fffdb41d..01ba9157c07 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -44,6 +44,7 @@ import org.apache.ambari.server.topology.ManagementPackMapping; import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.NoSuchBlueprintException; +import org.apache.ambari.server.topology.ProvisionRequest; import org.apache.ambari.server.topology.SecurityConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +57,7 @@ * Request for provisioning a cluster. */ @SuppressWarnings("unchecked") -public class ProvisionClusterRequest extends BaseClusterRequest { +public class ProvisionClusterRequest extends BaseClusterRequest implements ProvisionRequest { /** * host groups property name */ @@ -278,6 +279,7 @@ public void setClusterName(String clusterName) { this.clusterName = clusterName; } + @Override public ConfigRecommendationStrategy getConfigRecommendationStrategy() { return configRecommendationStrategy; } @@ -517,15 +519,18 @@ public String getQuickLinksProfileJson() { return quickLinksProfileJson; } + @Override public String getDefaultPassword() { return defaultPassword; } + @Override public Set getStackIds() { return stackIds; } - public Collection getMpackInstances() { + @Override + public Collection getMpacks() { return mpackInstances; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 761d59efbcc..c94495a165b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -226,7 +226,7 @@ public void createAmbariResources(ClusterTopology topology, String clusterName, String repoVersionString, Long repoVersionId) { Map repoVersionByStack = new HashMap<>(); - Set stackIds = topology.getBlueprint().getStackIds(); + Set stackIds = topology.getStackIds(); for (StackId stackId : stackIds) { RepositoryVersionEntity repoVersion = null; if (stackIds.size() == 1) { @@ -245,8 +245,7 @@ public void createAmbariResources(ClusterTopology topology, String clusterName, } } - StackId stackId = Iterables.getFirst(topology.getBlueprint().getStackIds(), null); - createAmbariClusterResource(clusterName, stackId, securityType); + createAmbariClusterResource(clusterName, topology.getStackIds(), securityType); createAmbariServiceAndComponentResources(topology, clusterName, repoVersionByStack); } @@ -330,19 +329,15 @@ private RepositoryVersionEntity findSpecifiedRepo(String repoVersionString, Long return repoVersion; } - public void createAmbariClusterResource(String clusterName, StackId stackId, SecurityType securityType) { - String stackInfo = stackId.toString(); + private void createAmbariClusterResource(String clusterName, Set stackIds, SecurityType securityType) { + String stackInfo = stackIds.iterator().next().toString(); // temporary final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType, stackInfo, null); try { - RetryHelper.executeWithRetry(new Callable() { - @Override - public Object call() throws Exception { - getController().createCluster(clusterRequest); - return null; - } + RetryHelper.executeWithRetry(() -> { + getController().createCluster(clusterRequest); + return null; }); - } catch (AmbariException e) { LOG.error("Failed to create Cluster resource: ", e); if (e.getCause() instanceof DuplicateResourceException) { @@ -353,37 +348,23 @@ public Object call() throws Exception { } } - public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { - Set serviceGroups = Sets.newHashSet(DEFAULT_SERVICE_GROUP_NAME); - Collection services = topology.getServices(); - - try { - Cluster cluster = getController().getClusters().getCluster(clusterName); - serviceGroups.removeAll(cluster.getServiceGroups().keySet()); - services.removeAll(cluster.getServices().keySet()); - } catch (AmbariException e) { - throw new RuntimeException("Failed to persist service and component resources: " + e, e); - } - - Set serviceGroupRequests = serviceGroups.stream() - .map(serviceGroupName -> new ServiceGroupRequest(clusterName, serviceGroupName)) + private void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { + Set serviceGroupRequests = topology.getComponents() + .map(c -> new ServiceGroupRequest(clusterName, c.getServiceGroupName())) .collect(toSet()); - Set serviceRequests = new HashSet<>(); - Set componentRequests = new HashSet<>(); - for (String service : services) { - String credentialStoreEnabled = topology.getBlueprint().getSetting().getCredentialStoreEnabled(service); - StackId stackId = Iterables.getOnlyElement(topology.getStackIdsForService(service)); // FIXME temporarily assume each service is defined in only one mpack - Long repositoryVersionId = repoVersionByStack.get(stackId); - serviceRequests.add(new ServiceRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, service, - repositoryVersionId, null, credentialStoreEnabled, stackId - )); + Set serviceRequests = topology.getComponents() + .map(c -> new ServiceRequest( + clusterName, c.getServiceGroupName(), c.getServiceName(), c.getServiceType(), repoVersionByStack.get(c.getStackId()), null, + topology.getSetting().getCredentialStoreEnabled(c.getServiceName()), // FIXME settings by service type or name? + c.getStackId() + )) + .collect(toSet()); - for (String component : topology.getComponentNames(service)) { - String recoveryEnabled = topology.getBlueprint().getSetting().getRecoveryEnabled(service, component); - componentRequests.add(new ServiceComponentRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, null, recoveryEnabled)); - } - } + Set componentRequests = topology.getComponents() + .map(c -> new ServiceComponentRequest(clusterName, c.getServiceGroupName(), c.getServiceName(), c.getComponentName(), null, + topology.getSetting().getRecoveryEnabled(c.getServiceName(), c.getComponentName()))) // FIXME settings by service type or name? + .collect(toSet()); try { if (!serviceGroupRequests.isEmpty()) { @@ -500,7 +481,7 @@ public static void init(HostRoleCommandFactory factory) { } public void registerHostWithConfigGroup(final String hostName, final ClusterTopology topology, final String groupName) { - String qualifiedGroupName = getConfigurationGroupName(topology.getBlueprint().getName(), groupName); + String qualifiedGroupName = getConfigurationGroupName(topology.getBlueprintName(), groupName); Lock configGroupLock = configGroupCreateLock.get(qualifiedGroupName); @@ -799,7 +780,7 @@ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String serviceConfigs.put(type, config); } - String bpName = topology.getBlueprint().getName(); + String bpName = topology.getBlueprintName(); for (Map.Entry> entry : groupConfigs.entrySet()) { String service = entry.getKey(); Map serviceConfigs = entry.getValue(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 3762df0b642..64456247601 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -19,7 +19,6 @@ package org.apache.ambari.server.topology; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Set; @@ -105,6 +104,9 @@ public interface Blueprint { */ BlueprintEntity toEntity(); - List getRepositorySettings(); + /** + * Add the kerberos client to all host groups in the blueprint. + */ + boolean ensureKerberosClientIsPresent(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java new file mode 100644 index 00000000000..43f372457ed --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static java.util.stream.Collectors.toMap; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; + +import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.orm.entities.BlueprintEntity; +import org.apache.ambari.server.state.SecurityType; +import org.apache.ambari.server.state.StackId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +/** + * I am the Blueprint and ProvisionClusterRequest combined. + */ +public class BlueprintBasedClusterProvisionRequest implements Blueprint, ProvisionRequest { + + private static final Logger LOG = LoggerFactory.getLogger(BlueprintBasedClusterProvisionRequest.class); + + private final Blueprint blueprint; + private final ProvisionClusterRequest request; + private final Set stackIds; + private final StackDefinition stack; + private final Map mpacks; + private final SecurityConfiguration securityConfiguration; + + public BlueprintBasedClusterProvisionRequest(AmbariContext ambariContext, SecurityConfigurationFactory securityConfigurationFactory, Blueprint blueprint, ProvisionClusterRequest request) { + this.blueprint = blueprint; + this.request = request; + + stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), request.getStackIds())); + stack = ambariContext.composeStacks(stackIds); + mpacks = ImmutableMap.copyOf( + Stream.concat(blueprint.getMpacks().stream(), request.getMpacks().stream()) + .collect(toMap(MpackInstance::getMpackName, Function.identity()))); + + securityConfiguration = processSecurityConfiguration(securityConfigurationFactory); + + if (securityConfiguration.getType() == SecurityType.KERBEROS) { + ensureKerberosClientIsPresent(); + } + } + + @Override + public String getName() { + return blueprint.getName(); + } + + @Override + public Map getHostGroups() { + return blueprint.getHostGroups(); + } + + @Override + public HostGroup getHostGroup(String name) { + return blueprint.getHostGroup(name); + } + + @Override + public Configuration getConfiguration() { + return request.getConfiguration(); + } + + @Override + public Setting getSetting() { + return blueprint.getSetting(); + } + + @Override + public Set getStackIds() { + return stackIds; + } + + @Override + public Collection getMpacks() { + return mpacks.values(); + } + + @Override + public Collection getHostGroupsForComponent(String component) { + return blueprint.getHostGroupsForComponent(component); + } + + @Nonnull + @Override + public SecurityConfiguration getSecurity() { + return securityConfiguration; + } + + @Override + public BlueprintEntity toEntity() { + throw new UnsupportedOperationException(); + } + + @Override + public Long getClusterId() { + return null; + } + + @Override + public Type getType() { + return Type.PROVISION; + } + + @Override + public Blueprint getBlueprint() { + return blueprint; + } + + @Override + public Map getHostGroupInfo() { + return request.getHostGroupInfo(); + } + + @Override + public String getDescription() { + return request.getDescription(); + } + + public String getDefaultPassword() { + return request.getDefaultPassword(); + } + + public ConfigRecommendationStrategy getConfigRecommendationStrategy() { + return request.getConfigRecommendationStrategy(); + } + + public ProvisionAction getProvisionAction() { + return request.getProvisionAction(); + } + + public StackDefinition getStack() { + return stack; + } + + public Map> getServicesByMpack() { + Map> result = new HashMap<>(); + for (MpackInstance mpack : mpacks.values()) { + Map services = mpack.getServiceInstances().stream() + .collect(toMap(ServiceInstance::getName, Function.identity())); + result.put(mpack.getMpackName(), services); + } + return result; + } + + /** + * @return service instances defined in the topology, mapped by service name, + * whose name is unique across all mpacks. + */ + public Map getUniqueServices() { + Map map = mpacks.values().stream() + .flatMap(mpack -> mpack.getServiceInstances().stream()) + .collect(toMap(ServiceInstance::getName, Function.identity(), (s1, s2) -> null)); + map.entrySet().removeIf(e -> e.getValue() == null); // remove non-unique names mapped to null + return map; + } + + /** + * Retrieve security info from Blueprint if missing from Cluster Template request. + */ + private SecurityConfiguration processSecurityConfiguration(SecurityConfigurationFactory securityConfigurationFactory) { + SecurityConfiguration blueprintSecurity = blueprint.getSecurity(); + SecurityConfiguration requestSecurity = request.getSecurityConfiguration(); + + if (requestSecurity == null) { + LOG.debug("There's no security configuration in the request, retrieving it from the associated blueprint"); + requestSecurity = blueprintSecurity; + if (requestSecurity.getType() == SecurityType.KERBEROS && requestSecurity.getDescriptorReference() != null) { + requestSecurity = securityConfigurationFactory.loadSecurityConfigurationByReference(requestSecurity.getDescriptorReference()); + } + } else if (requestSecurity.getType() == SecurityType.NONE && blueprintSecurity.getType() == SecurityType.KERBEROS) { + throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); + } + + return requestSecurity; + } + + @Override + public boolean ensureKerberosClientIsPresent() { + return blueprint.ensureKerberosClientIsPresent(); + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index e2fab706899..47e19bafbe7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -436,9 +436,15 @@ private List processRepoSettings() { return setting != null ? setting.processRepoSettings() : Collections.emptyList(); } - @Override - public List getRepositorySettings(){ - return repoSettings; + /** + * Add the Kerberos client to all host groups. + */ + public boolean ensureKerberosClientIsPresent() { + boolean changed = false; + for (HostGroup group : getHostGroups().values()) { + changed |= group.addComponent(new Component("KERBEROS_CLIENT")); + } + return changed; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 05fe8390bec..54e28e38806 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -18,6 +18,10 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -166,7 +170,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map componentHostsMap = createComponentHostMap(blueprint); + Map componentHostsMap = createComponentHostMap(); existingConfigurations.put("clusterHostInfo", componentHostsMap); try { @@ -178,7 +182,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> updatedConfigs = AmbariContext.getController().getKerberosHelper() .getServiceConfigurationUpdates(cluster, existingConfigurations, - createServiceComponentMap(blueprint), null, null, true, false); + createServiceComponentMap(), null, null, true, false); // ****************************************************************************************** // Since Kerberos is being enabled, make sure the cluster-env/security_enabled property is @@ -225,21 +229,11 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> createServiceComponentMap(Blueprint blueprint) { - Map> serviceComponents = new HashMap<>(); - Collection services = clusterTopology.getServices(); - - if(services != null) { - for (String service : services) { - Collection components = clusterTopology.getComponentNames(service); - serviceComponents.put(service, components == null ? Collections.emptySet() : new HashSet<>(components)); - } - } - - return serviceComponents; + private Map> createServiceComponentMap() { + return clusterTopology.getComponents() + .collect(groupingBy(ResolvedComponent::getServiceName, mapping(ResolvedComponent::getComponentName, toSet()))); } /** @@ -272,19 +266,17 @@ private boolean propertyHasCustomValue(Map clusterConfigProperti return propertyHasCustomValue; } - private Map createComponentHostMap(Blueprint blueprint) { + private Map createComponentHostMap() { Map componentHostsMap = new HashMap<>(); - for (String service : clusterTopology.getServices()) { - Collection components = clusterTopology.getComponentNames(service); - for (String component : components) { - Collection componentHost = clusterTopology.getHostAssignmentsForComponent(component); - // retrieve corresponding clusterInfoKey for component using StageUtils - String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component); - if (clusterInfoKey == null) { - clusterInfoKey = component.toLowerCase() + "_hosts"; - } - componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); + for (ResolvedComponent component : clusterTopology.getComponents().collect(toSet())) { + String componentName = component.getComponentName(); + Collection componentHost = clusterTopology.getHostAssignmentsForComponent(componentName); + // retrieve corresponding clusterInfoKey for component using StageUtils + String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(componentName); + if (clusterInfoKey == null) { + clusterInfoKey = componentName.toLowerCase() + "_hosts"; } + componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); } return componentHostsMap; } @@ -294,7 +286,6 @@ private Collection getRequiredHostgroupsForKerberosConfiguration() { try { Cluster cluster = getCluster(); - Blueprint blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); Map> existingConfigurations = clusterConfiguration.getFullProperties(); @@ -303,7 +294,7 @@ private Collection getRequiredHostgroupsForKerberosConfiguration() { // apply Kerberos specific configurations Map> updatedConfigs = AmbariContext.getController().getKerberosHelper() .getServiceConfigurationUpdates(cluster, existingConfigurations, - createServiceComponentMap(blueprint), null, null, true, false); + createServiceComponentMap(), null, null, true, false); // retrieve hostgroup for component names extracted from variables like "{clusterHostInfo.(component_name) // _host}" diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index 9e31c648dad..67ee0070bb5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -23,6 +23,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.StackDefinition; @@ -42,18 +44,18 @@ public interface ClusterTopology { Long getClusterId(); /** - * Set the id of the cluster. + * Get the blueprint associated with the cluster. * - * @param clusterId cluster id + * @return associated blueprint */ - void setClusterId(Long clusterId); + Blueprint getBlueprint(); /** - * Get the blueprint associated with the cluster. + * Get the name of the blueprint associated with the cluster. * - * @return associated blueprint + * @return associated blueprint's name */ - Blueprint getBlueprint(); + String getBlueprintName(); /** * Get the stack associated with the blueprint. @@ -79,6 +81,15 @@ public interface ClusterTopology { */ Configuration getConfiguration(); + /** + * Get the Blueprint cluster scoped setting. + * The blueprint cluster scoped setting has the setting properties + * with the setting names associated with the blueprint. + * + * @return blueprint cluster scoped setting + */ + Setting getSetting(); + /** * Get host group information. * @@ -93,6 +104,7 @@ public interface ClusterTopology { * * @return collection of host group names which contain the specified component */ + @Deprecated // 1. component name is not enough, 2. only used for stack-specific checks/updates Collection getHostGroupsForComponent(String component); /** @@ -113,6 +125,7 @@ public interface ClusterTopology { * * @return collection of hosts for the specified component; will not return null */ + @Deprecated Collection getHostAssignmentsForComponent(String component); /** @@ -122,58 +135,21 @@ public interface ClusterTopology { */ Collection getServices(); - /** - * @return the mpack instance defined in the topology with the given name - */ - MpackInstance getMpack(String name); - - Map> getServicesByMpack(); - - /** - * @return service instances defined in the topology, mapped by service name, - * whose name is unique across all mpacks. - */ - Map getUniqueServices(); - /** * Get all of the components represented in the blueprint. * * @return collection of all represented components */ - Stream getComponents(); + Stream getComponents(); /** - * Get the names of all of the components represented in the blueprint. + * Get the components that are included in the specified host group. * - * @return collection of all represented component names + * @param hostGroup host group name + * @return stream of components for the service */ - Stream getComponentNames(); - - /** - * Look up the stacks that define service. - * To be used only after checking that services map to - * @param service the name of the service as defined in the stack (mpack), eg. ZOOKEEPER - * @return the ID of the stack that defines the given service - */ - Set getStackIdsForService(String service); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of components for the service. Will not return null. - */ - Collection getComponents(String service); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of component names for the service. Will not return null. - */ - Collection getComponentNames(String service); + @Nonnull + Stream getComponentsInHostGroup(String hostGroup); /** * A config type is valid if there are services related to except cluster-env and global. @@ -201,20 +177,6 @@ public interface ClusterTopology { */ void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException; - /** - * Determine if NameNode HA is enabled. - * - * @return true if NameNode HA is enabled; false otherwise - */ - boolean isNameNodeHAEnabled(); - - /** - * Determine if Yarn ResourceManager HA is enabled. - * - * @return true if Yarn ResourceManager HA is enabled; false otherwise - */ - boolean isYarnResourceManagerHAEnabled(); - /** * Determine if the cluster is kerberos enabled. * @@ -239,16 +201,8 @@ public interface ClusterTopology { */ RequestStatusResponse startHost(String hostName, boolean skipFailure); - void setConfigRecommendationStrategy(ConfigRecommendationStrategy strategy); - ConfigRecommendationStrategy getConfigRecommendationStrategy(); - /** - * Set request provision action : INSTALL vs INSTALL_AND_START - * @param provisionAction @ProvisionAction - */ - void setProvisionAction(ProvisionAction provisionAction); - ProvisionAction getProvisionAction(); Map getAdvisedConfigurations(); @@ -269,7 +223,7 @@ public interface ClusterTopology { * * @return true if the host group contains a master component; false otherwise */ - boolean containsMasterComponent(HostGroup hostGroup); + boolean containsMasterComponent(String hostGroup); - Iterable getHostGroups(); + Collection getHostGroups(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index 59ecc021351..e9e9e26b33b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -19,8 +19,6 @@ package org.apache.ambari.server.topology; -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; @@ -31,14 +29,13 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.internal.BaseClusterRequest; import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; @@ -47,7 +44,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; /** * Represents a cluster topology. @@ -62,50 +58,53 @@ public class ClusterTopologyImpl implements ClusterTopology { private Long clusterId; private final Blueprint blueprint; private final Configuration configuration; - private ConfigRecommendationStrategy configRecommendationStrategy; - private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START; + private final ConfigRecommendationStrategy configRecommendationStrategy; + private final ProvisionAction provisionAction; private final Map advisedConfigurations = new HashMap<>(); private final Map hostGroupInfoMap = new HashMap<>(); private final AmbariContext ambariContext; + private final BlueprintBasedClusterProvisionRequest provisionRequest; private final String defaultPassword; - private final Map mpacks; + private final Map> resolvedComponents; public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topologyRequest) throws InvalidTopologyException { this.ambariContext = ambariContext; this.clusterId = topologyRequest.getClusterId(); - // provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here this.blueprint = topologyRequest.getBlueprint(); this.configuration = topologyRequest.getConfiguration(); + configRecommendationStrategy = ConfigRecommendationStrategy.NEVER_APPLY; + provisionAction = topologyRequest instanceof BaseClusterRequest ? ((BaseClusterRequest) topologyRequest).getProvisionAction() : INSTALL_AND_START; // FIXME - // FIXME smelling more and more - // 1. shouldn't have to cast - // 2. what about replayed request - // -> - // 1. interface to combine blueprint and provision request - // 2. replayed request should simply be a provision or scale request - // 3. do not create a ClusterTopologyImpl for scale request -- create for original provision request only - if (topologyRequest instanceof ProvisionClusterRequest) { - ProvisionClusterRequest provisionRequest = (ProvisionClusterRequest) topologyRequest; - defaultPassword = provisionRequest.getDefaultPassword(); - stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), provisionRequest.getStackIds())); - stack = ambariContext.composeStacks(stackIds); - setBlueprintParentConfig(); - mpacks = ImmutableMap.copyOf(Stream.concat(blueprint.getMpacks().stream(), provisionRequest.getMpackInstances().stream()) - .collect(toMap( - MpackInstance::getMpackName, - Function.identity()))); - } else { - defaultPassword = null; - stackIds = ImmutableSet.of(); - stack = null; - mpacks = ImmutableMap.of(); - } + provisionRequest = null; + defaultPassword = null; + stackIds = ImmutableSet.of(); + stack = null; + resolvedComponents = ImmutableMap.of(); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); } - void setBlueprintParentConfig() { + // FIXME 2. replayed request should simply be a provision or scale request + // FIXME 3. do not create a ClusterTopologyImpl for scale request -- create for original provision request only + public ClusterTopologyImpl( + AmbariContext ambariContext, + BlueprintBasedClusterProvisionRequest request, + Map> resolvedComponents + ) throws InvalidTopologyException { + this.ambariContext = ambariContext; + this.blueprint = request.getBlueprint(); + this.configuration = request.getConfiguration(); + this.provisionRequest = request; + this.resolvedComponents = resolvedComponents; + configRecommendationStrategy = request.getConfigRecommendationStrategy(); + provisionAction = request.getProvisionAction(); + + defaultPassword = provisionRequest.getDefaultPassword(); + stackIds = request.getStackIds(); + stack = request.getStack(); + blueprint.getConfiguration().setParentConfiguration(stack.getConfiguration(getServices())); + registerHostGroupInfo(request.getHostGroupInfo()); } @Override @@ -118,7 +117,6 @@ public Long getClusterId() { return clusterId; } - @Override public void setClusterId(Long clusterId) { this.clusterId = clusterId; } @@ -128,6 +126,11 @@ public Blueprint getBlueprint() { return blueprint; } + @Override + public String getBlueprintName() { + return blueprint.getName(); + } + @Override public Set getStackIds() { return stackIds; @@ -143,30 +146,32 @@ public Configuration getConfiguration() { return configuration; } + @Override + public Setting getSetting() { + return provisionRequest.getSetting(); + } + @Override public Map getHostGroupInfo() { return hostGroupInfoMap; } @Override - public Iterable getHostGroups() { + public Collection getHostGroups() { return blueprint.getHostGroups().values(); } - //todo: do we want to return groups with no requested hosts? @Override public Collection getHostGroupsForComponent(String component) { - Collection resultGroups = new ArrayList<>(); - for (HostGroup group : getBlueprint().getHostGroups().values() ) { - if (group.getComponentNames().contains(component)) { - resultGroups.add(group.getName()); - } - } - return resultGroups; + return resolvedComponents.entrySet().stream() + .filter(e -> e.getValue().stream().anyMatch(c -> component.equals(c.getComponentName()))) + .map(Map.Entry::getKey) + .collect(toSet()); } @Override public String getHostGroupForHost(String hostname) { + // FIXME change to map lookup for (HostGroupInfo groupInfo : hostGroupInfoMap.values() ) { if (groupInfo.getHostNames().contains(hostname)) { // a host can only be associated with a single host group @@ -223,71 +228,26 @@ public Collection getHostAssignmentsForComponent(String component) { @Override public Collection getServices() { - return getComponentNames() - .map(stack::getServiceForComponent) - .collect(toSet()); - } - - @Override - public MpackInstance getMpack(String name) { - return mpacks.get(name); - } - - @Override - public Map> getServicesByMpack() { - Map> result = new HashMap<>(); - for (MpackInstance mpack : mpacks.values()) { - Map services = mpack.getServiceInstances().stream() - .collect(toMap(ServiceInstance::getName, Function.identity())); - result.put(mpack.getMpackName(), services); - } - return result; - } - - @Override - public Map getUniqueServices() { - Map map = mpacks.values().stream() - .flatMap(mpack -> mpack.getServiceInstances().stream()) - .collect(toMap(ServiceInstance::getName, Function.identity(), (s1, s2) -> null)); - map.entrySet().removeIf(e -> e.getValue() == null); // remove non-unique names mapped to null - return map; - } - - @Override - public Stream getComponentNames() { return getComponents() - .map(Component::getName); - } - - @Override - public Stream getComponents() { - return getBlueprint().getHostGroups().values().stream() - .flatMap(hostGroup -> hostGroup.getComponents().stream()); - } - - @Override - public Set getStackIdsForService(String service) { - return stack.getStacksForService(service); + .map(ResolvedComponent::getServiceName) + .collect(toSet()); } @Override - public Collection getComponents(String service) { - return getComponents() - .filter(c -> stack.getComponents(service).contains(c.getName())) - .collect(toSet()); + public Stream getComponents() { + return resolvedComponents.values().stream() + .flatMap(Collection::stream); } @Override - public Collection getComponentNames(String service) { - return getComponents(service).stream().map(Component::getName).collect(toList()); + public Stream getComponentsInHostGroup(String hostGroup) { + return resolvedComponents.computeIfAbsent(hostGroup, __ -> ImmutableSet.of()).stream(); } @Override - public boolean containsMasterComponent(HostGroup hostGroup) { - return hostGroup.getComponentNames().stream() - .map(stack::isMasterComponent) - .findAny() - .orElse(false); + public boolean containsMasterComponent(String hostGroup) { + return resolvedComponents.getOrDefault(hostGroup, ImmutableSet.of()).stream() + .anyMatch(ResolvedComponent::isMasterComponent); } @Override @@ -303,38 +263,11 @@ public boolean isValidConfigType(String configType) { } } - @Override - public boolean isNameNodeHAEnabled() { - return isNameNodeHAEnabled(configuration.getFullProperties()); - } - - public static boolean isNameNodeHAEnabled(Map> configurationProperties) { - return configurationProperties.containsKey("hdfs-site") && - (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") || - configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices")); - } - - @Override - public boolean isYarnResourceManagerHAEnabled() { - return isYarnResourceManagerHAEnabled(configuration.getFullProperties()); - } - - /** - * Static convenience function to determine if Yarn ResourceManager HA is enabled - * @param configProperties configuration properties for this cluster - * @return true if Yarn ResourceManager HA is enabled - * false if Yarn ResourceManager HA is not enabled - */ - static boolean isYarnResourceManagerHAEnabled(Map> configProperties) { - return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled") - && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true"); - } - // FIXME move out private void validateTopology() throws InvalidTopologyException { - if(isNameNodeHAEnabled()){ + if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(getConfiguration().getFullProperties())) { Collection nnHosts = getHostAssignmentsForComponent("NAMENODE"); if (nnHosts.size() != 2) { throw new InvalidTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " + @@ -398,11 +331,6 @@ public RequestStatusResponse startHost(String hostName, boolean skipFailure) { } } - @Override - public void setConfigRecommendationStrategy(ConfigRecommendationStrategy strategy) { - this.configRecommendationStrategy = strategy; - } - @Override public ConfigRecommendationStrategy getConfigRecommendationStrategy() { return this.configRecommendationStrategy; @@ -413,11 +341,6 @@ public ProvisionAction getProvisionAction() { return provisionAction; } - @Override - public void setProvisionAction(ProvisionAction provisionAction) { - this.provisionAction = provisionAction; - } - @Override public Map getAdvisedConfigurations() { return this.advisedConfigurations; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java index 25d0c2b3056..9aefa91fce9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java @@ -44,23 +44,22 @@ public class ComponentResolver { private static final Logger LOG = LoggerFactory.getLogger(ComponentResolver.class); - private final ClusterTopology topology; + private final BlueprintBasedClusterProvisionRequest request; private final Map uniqueServices; private final Map> mpackServices; - public ComponentResolver(ClusterTopology topology) { - this.topology = topology; - - uniqueServices = topology.getUniqueServices(); - mpackServices = topology.getServicesByMpack(); + public ComponentResolver(BlueprintBasedClusterProvisionRequest request) { + this.request = request; + uniqueServices = request.getUniqueServices(); + mpackServices = request.getServicesByMpack(); } public Map> resolve() { Map> result = new HashMap<>(); List problems = new LinkedList<>(); - StackDefinition stack = topology.getStack(); - for (HostGroup hg : topology.getHostGroups()) { + StackDefinition stack = request.getStack(); + for (HostGroup hg : request.getHostGroups().values()) { result.put(hg.getName(), new HashSet<>()); for (Component comp : hg.getComponents()) { @@ -130,8 +129,7 @@ private Stream> filterByServiceName(Component comp, Stream if (service != null) { String serviceType = service.getType(); - return stream - .filter(pair -> pair.getRight().equals(serviceType)); + return stream.filter(pair -> pair.getRight().equals(serviceType)); } } @@ -141,8 +139,7 @@ private Stream> filterByServiceName(Component comp, Stream // if component references a specific mpack instance, filter the stream by the name of that mpack private Stream> filterByMpackName(Component comp, Stream> stream) { if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { - return stream - .filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); + return stream.filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); } return stream; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java index 5e3874b5913..a8ecc12d2cf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java @@ -98,7 +98,7 @@ public HostRequest(long requestId, long id, long clusterId, String hostname, Str this.hostGroup = hostGroup; hostgroupName = hostGroup.getName(); this.predicate = predicate; - containsMaster = topology.containsMasterComponent(hostGroup); + containsMaster = topology.containsMasterComponent(hostgroupName); this.topology = topology; this.skipFailure = skipFailure; createTasks(this.skipFailure); @@ -121,14 +121,14 @@ public HostRequest(long requestId, long id, String predicate, this.requestId = requestId; this.id = id; clusterId = topology.getClusterId(); - blueprint = topology.getBlueprint().getName(); + blueprint = topology.getBlueprintName(); hostgroupName = entity.getTopologyHostGroupEntity().getName(); hostGroup = topology.getBlueprint().getHostGroup(hostgroupName); hostname = entity.getHostName(); setStatus(entity.getStatus()); statusMessage = entity.getStatusMessage(); this.predicate = toPredicate(predicate); - containsMaster = topology.containsMasterComponent(hostGroup); + containsMaster = topology.containsMasterComponent(hostgroupName); this.topology = topology; this.skipFailure = skipFailure; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java index b23aa25e649..39c711a426f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java @@ -444,7 +444,7 @@ public Optional getFailureReason() { private void createHostRequests(TopologyRequest request, ClusterTopology topology) { Map hostGroupInfoMap = request.getHostGroupInfo(); Blueprint blueprint = topology.getBlueprint(); - boolean skipFailure = blueprint.getSetting().shouldSkipFailure(); + boolean skipFailure = topology.getSetting().shouldSkipFailure(); for (HostGroupInfo hostGroupInfo : hostGroupInfoMap.values()) { String groupName = hostGroupInfo.getHostGroupName(); int hostCardinality = hostGroupInfo.getRequestedHostCount(); @@ -455,14 +455,14 @@ private void createHostRequests(TopologyRequest request, ClusterTopology topolog // host names are specified String hostname = hostnames.get(i); HostRequest hostRequest = new HostRequest(getRequestId(), hostIdCounter.getAndIncrement(), getClusterId(), - hostname, blueprint.getName(), blueprint.getHostGroup(groupName), null, topology, skipFailure); + hostname, topology.getBlueprintName(), blueprint.getHostGroup(groupName), null, topology, skipFailure); synchronized (requestsWithReservedHosts) { requestsWithReservedHosts.put(hostname, hostRequest); } } else { // host count is specified HostRequest hostRequest = new HostRequest(getRequestId(), hostIdCounter.getAndIncrement(), getClusterId(), - null, blueprint.getName(), blueprint.getHostGroup(groupName), hostGroupInfo.getPredicate(), topology, skipFailure); + null, topology.getBlueprintName(), blueprint.getHostGroup(groupName), hostGroupInfo.getPredicate(), topology, skipFailure); outstandingHostRequests.add(hostRequest); } } @@ -495,7 +495,7 @@ private void createHostRequests(ClusterTopology topology, } } - boolean skipFailure = topology.getBlueprint().getSetting().shouldSkipFailure(); + boolean skipFailure = topology.getSetting().shouldSkipFailure(); for (TopologyHostRequestEntity hostRequestEntity : requestEntity.getTopologyHostRequestEntities()) { Long hostRequestId = hostRequestEntity.getId(); synchronized (hostIdCounter) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java index bb1f85234d7..13c9eff95e6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java @@ -31,6 +31,7 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.api.predicate.InvalidQueryException; import org.apache.ambari.server.controller.internal.BaseClusterRequest; +import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.orm.dao.HostDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.orm.dao.TopologyHostGroupDAO; @@ -215,9 +216,6 @@ public Map> getAllRequests() { if (clusterTopology == null) { try { clusterTopology = new ClusterTopologyImpl(ambariContext, replayedRequest); - if (entity.getProvisionAction() != null) { - clusterTopology.setProvisionAction(entity.getProvisionAction()); - } topologyRequests.put(replayedRequest.getClusterId(), clusterTopology); allRequests.put(clusterTopology, new ArrayList<>()); } catch (InvalidTopologyException e) { @@ -392,11 +390,13 @@ private static class ReplayedTopologyRequest implements TopologyRequest { private final Blueprint blueprint; private final Configuration configuration; private final Map hostGroupInfoMap = new HashMap<>(); + private final ProvisionAction provisionAction; public ReplayedTopologyRequest(TopologyRequestEntity entity, BlueprintFactory blueprintFactory) { clusterId = entity.getClusterId(); type = Type.valueOf(entity.getAction()); description = entity.getDescription(); + provisionAction = entity.getProvisionAction(); try { blueprint = blueprintFactory.getBlueprint(entity.getBlueprintName()); @@ -451,6 +451,10 @@ private Configuration createConfiguration(String propString, String attributeStr return new Configuration(properties, attributes); } + public ProvisionAction getProvisionAction() { + return provisionAction; + } + private void parseHostGroupInfo(TopologyRequestEntity entity) { for (TopologyHostGroupEntity hostGroupEntity : entity.getTopologyHostGroupEntities()) { for (TopologyHostInfoEntity hostInfoEntity : hostGroupEntity.getTopologyHostInfoEntities()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java new file mode 100644 index 00000000000..46bad4ebde8 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java @@ -0,0 +1,17 @@ +package org.apache.ambari.server.topology; + +import java.util.Collection; +import java.util.Set; + +import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.state.StackId; + +public interface ProvisionRequest extends TopologyRequest { + + ConfigRecommendationStrategy getConfigRecommendationStrategy(); + ProvisionAction getProvisionAction(); + String getDefaultPassword(); + Set getStackIds(); + Collection getMpacks(); + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java index 259c27523ac..ceeacd48196 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -45,8 +45,19 @@ public String getServiceType() { return serviceType; } + public String getComponentName() { + return component.getName(); + } + public Component getComponent() { return component; } + public String getServiceGroupName() { + return stackId.toString(); // FIXME + } + + public boolean isMasterComponent() { + return false; // FIXME + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 39ea3bbdc34..1ce8131d37b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -208,7 +208,7 @@ private void ensureInitialized() { // ensure KERBEROS_CLIENT is present in each hostgroup even if it's not in original BP for(ClusterTopology clusterTopology : clusterTopologyMap.values()) { if (clusterTopology.isClusterKerberosEnabled()) { - addKerberosClient(clusterTopology); + clusterTopology.getBlueprint().ensureKerberosClientIsPresent(); } } isInitialized = true; @@ -236,12 +236,12 @@ public void onRequestFinished(RequestFinishedEvent event) { if(isLogicalRequestSuccessful(provisionRequest)) { LOG.info("Cluster creation request id={} using Blueprint {} successfully completed for cluster id={}", clusterProvisionWithBlueprintCreateRequests.get(event.getClusterId()).getRequestId(), - clusterTopologyMap.get(event.getClusterId()).getBlueprint().getName(), + clusterTopologyMap.get(event.getClusterId()).getBlueprintName(), event.getClusterId()); } else { LOG.info("Cluster creation request id={} using Blueprint {} failed for cluster id={}", clusterProvisionWithBlueprintCreateRequests.get(event.getClusterId()).getRequestId(), - clusterTopologyMap.get(event.getClusterId()).getBlueprint().getName(), + clusterTopologyMap.get(event.getClusterId()).getBlueprintName(), event.getClusterId()); } } @@ -277,33 +277,28 @@ public boolean isClusterProvisionWithBlueprintFinished(long clusterId) { public RequestStatusResponse provisionCluster(final ProvisionClusterRequest request) throws InvalidTopologyException, AmbariException { ensureInitialized(); - final ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, request); + BlueprintBasedClusterProvisionRequest provisionRequest = new BlueprintBasedClusterProvisionRequest(ambariContext, securityConfigurationFactory, request.getBlueprint(), request); + Map> resolved = new ComponentResolver(provisionRequest).resolve(); + + final ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, provisionRequest, resolved); final String clusterName = request.getClusterName(); final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); + final SecurityConfiguration securityConfiguration = provisionRequest.getSecurity(); - SecurityConfiguration securityConfiguration = processSecurityConfiguration(request); - SecurityType securityType = securityConfiguration.getType(); - if (securityType == SecurityType.KERBEROS && addKerberosClient(topology)) { - // refresh default stack config after adding KERBEROS_CLIENT component to topology - topology.setBlueprintParentConfig(); - } - - Map> resolved = new ComponentResolver(topology).resolve(); - // FIXME use resolved components in topology topologyValidatorService.validateTopologyConfiguration(topology); // FIXME known stacks validation is too late here // get the id prior to creating ambari resources which increments the counter final Long provisionId = ambariContext.getNextRequestId(); // create resources - ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID); + ambariContext.createAmbariResources(topology, clusterName, securityConfiguration.getType(), repoVersion, repoVersionID); if (securityConfiguration.getDescriptor() != null) { submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor()); } - if (securityType == SecurityType.KERBEROS) { + if (securityConfiguration.getType() == SecurityType.KERBEROS) { Credential credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); submitCredential(clusterName, credential); } @@ -311,11 +306,6 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ long clusterId = ambariContext.getClusterId(clusterName); topology.setClusterId(clusterId); request.setClusterId(clusterId); - // set recommendation strategy - topology.setConfigRecommendationStrategy(request.getConfigRecommendationStrategy()); - // set provision action requested - topology.setProvisionAction(request.getProvisionAction()); - // create task executor for TopologyTasks getOrCreateTopologyTaskExecutor(clusterId); @@ -333,7 +323,7 @@ public LogicalRequest call() throws Exception { clusterTopologyMap.put(clusterId, topology); addClusterConfigRequest(logicalRequest, topology, new ClusterConfigurationRequest(ambariContext, topology, true, - stackAdvisorBlueprintProcessor, securityType == SecurityType.KERBEROS)); + stackAdvisorBlueprintProcessor, securityConfiguration.getType() == SecurityType.KERBEROS)); // Process the logical request processRequest(request, topology, logicalRequest); @@ -341,7 +331,7 @@ public LogicalRequest call() throws Exception { //todo: this should be invoked as part of a generic lifecycle event which could possibly //todo: be tied to cluster state - StackId stackId = Iterables.getFirst(topology.getBlueprint().getStackIds(), null); // FIXME need for stackId in ClusterRequest will be removed + StackId stackId = Iterables.getFirst(topology.getStackIds(), null); // FIXME need for stackId in ClusterRequest will be removed ambariContext.persistInstallStateForUI(clusterName, stackId); clusterProvisionWithBlueprintCreateRequests.put(clusterId, logicalRequest); return getRequestStatus(logicalRequest.getRequestId()); @@ -413,26 +403,6 @@ private void submitCredential(String clusterName, Credential credential) { } - /** - * Retrieve security info from Blueprint if missing from Cluster Template request. - */ - private SecurityConfiguration processSecurityConfiguration(ProvisionClusterRequest request) { - SecurityConfiguration securityConfiguration = request.getSecurityConfiguration(); - SecurityConfiguration blueprintSecurity = request.getBlueprint().getSecurity(); - - if (securityConfiguration == null) { - LOG.debug("There's no security configuration in the request, retrieving it from the associated blueprint"); - securityConfiguration = blueprintSecurity; - if (securityConfiguration.getType() == SecurityType.KERBEROS && securityConfiguration.getDescriptorReference() != null) { - securityConfiguration = securityConfigurationFactory.loadSecurityConfigurationByReference(securityConfiguration.getDescriptorReference()); - } - } else if (securityConfiguration.getType() == SecurityType.NONE && blueprintSecurity.getType() == SecurityType.KERBEROS) { - throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); - } - - return securityConfiguration; - } - private void submitKerberosDescriptorAsArtifact(String clusterName, String descriptor) { ResourceProvider artifactProvider = @@ -840,8 +810,6 @@ private void processRequest(TopologyRequest request, ClusterTopology topology, f LOG.info("TopologyManager.processRequest: Entering"); - finalizeTopology(request, topology); - boolean requestHostComplete = false; //todo: overall synchronization. Currently we have nested synchronization here @@ -1072,27 +1040,10 @@ private boolean isLogicalRequestSuccessful(LogicalRequest logicalRequest) { return logicalRequest != null && logicalRequest.isSuccessful(); } - //todo: this should invoke a callback on each 'service' in the topology - private void finalizeTopology(TopologyRequest request, ClusterTopology topology) { - } - private boolean isHostIgnored(String host) { return hostsToIgnore.remove(host); } - /** - * Add the kerberos client to groups if kerberos is enabled for the cluster. - * - * @param topology cluster topology - */ - private boolean addKerberosClient(ClusterTopology topology) { - boolean changed = false; - for (HostGroup group : topology.getBlueprint().getHostGroups().values()) { - changed |= group.addComponent(new Component("KERBEROS_CLIENT")); - } - return changed; - } - /** * Register the configuration task which is responsible for configuration topology resolution * and setting the updated configuration on the cluster. This task needs to be submitted to the diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java index bd5630b9875..94fcf23e3b1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java @@ -43,10 +43,6 @@ enum Type { PROVISION, SCALE, EXPORT } */ Type getType(); - //todo: only a single BP may be specified so all host groups have the same bp. - //todo: BP really needs to be associated with the HostGroupInfo, even for create which will have a single BP - //todo: for all HG's. - /** * Get the blueprint instance associated with the request. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java index d4b931c8075..a25dcc4ce45 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java @@ -18,14 +18,10 @@ package org.apache.ambari.server.topology.validators; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.StackDefinition; @@ -35,7 +31,6 @@ import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.ClusterTopologyImpl; import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.InvalidTopologyException; @@ -52,7 +47,7 @@ public class DependencyAndCardinalityValidator implements TopologyValidator { @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { Blueprint blueprint = topology.getBlueprint(); - LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName()); + LOGGER.info("Validating topology for blueprint: [{}]", topology.getBlueprintName()); StackDefinition stack = topology.getStack(); Collection hostGroups = blueprint.getHostGroups().values(); @@ -76,7 +71,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { cardinalityFailures.addAll(verifyComponentInAllHostGroups(blueprint, new Component(component), autoDeploy)); } else { cardinalityFailures.addAll(verifyComponentCardinalityCount( - stack, blueprint, new Component(component), cardinality, autoDeploy)); + stack, topology, blueprint, new Component(component), cardinality, autoDeploy)); } } } @@ -86,45 +81,6 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { } } - public void validateRequiredProperties(ClusterTopology topology) throws InvalidTopologyException { - // we don't want to include default stack properties so we can't just use hostGroup full properties - Map> clusterConfigurations = topology.getConfiguration().getProperties(); - - for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { - Map> operationalConfiguration = new HashMap<>(clusterConfigurations); - - operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); - for (String component : hostGroup.getComponentNames()) { - if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { - Map hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); - if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - List hostGroupsForComponent = new ArrayList<>(topology.getBlueprint().getHostGroupsForComponent(component)); - Set givenHostGroups = new HashSet<>(); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); - if(givenHostGroups.size() != hostGroupsForComponent.size()) { - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - if (BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()) { - for (HostGroup hostGroupForComponent : hostGroupsForComponent) { - Iterator itr = givenHostGroups.iterator(); - while(itr.hasNext()){ - if(itr.next().contains(hostGroupForComponent.getName())){ - itr.remove(); - } - } - } - } - - if(!givenHostGroups.isEmpty()){ - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - } - } - } - } - } - /** * Verify that a component is included in all host groups. * For components that are auto-install enabled, will add component to topology if needed. @@ -166,7 +122,7 @@ private Map> validateHostGroup(ClusterTopolog boolean isClientDependency = stack.getComponentInfo(dependency.getComponentName()).isClient(); if (isClientDependency && !topology.getServices().contains(dependency.getServiceName())) { LOGGER.debug("The service [{}] for component [{}] is missing from the blueprint [{}], skipping dependency", - dependency.getServiceName(), dependency.getComponentName(), topology.getBlueprint().getName()); + dependency.getServiceName(), dependency.getComponentName(), topology.getBlueprintName()); continue; } @@ -179,7 +135,7 @@ private Map> validateHostGroup(ClusterTopolog if(dependency.hasDependencyConditions()) { boolean conditionsSatisfied = true; for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions()) { - if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties())) { + if (!dependencyCondition.isResolved(topology.getConfiguration().getFullProperties())) { conditionsSatisfied = false; break; } @@ -190,7 +146,7 @@ private Map> validateHostGroup(ClusterTopolog } if (dependencyScope.equals("cluster")) { Collection missingDependencyInfo = verifyComponentCardinalityCount( - stack, blueprint, new Component(componentName), new Cardinality("1+"), autoDeployInfo); + stack, topology, blueprint, new Component(componentName), new Cardinality("1+"), autoDeployInfo); resolved = missingDependencyInfo.isEmpty(); } else if (dependencyScope.equals("host")) { @@ -228,15 +184,16 @@ private Map> validateHostGroup(ClusterTopolog */ private Collection verifyComponentCardinalityCount( StackDefinition stack, + ClusterTopology topology, Blueprint blueprint, Component component, Cardinality cardinality, AutoDeployInfo autoDeploy ) { - Map> configProperties = blueprint.getConfiguration().getProperties(); + Map> configProperties = topology.getConfiguration().getProperties(); Collection cardinalityFailures = new HashSet<>(); //todo: don't hard code this HA logic here - if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && + if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(configProperties) && (component.getName().equals("SECONDARY_NAMENODE"))) { // override the cardinality for this component in an HA deployment, // since the SECONDARY_NAMENODE should not be started in this scenario diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index 1a9bf07dda3..da6f8044a48 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -51,7 +51,8 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { } boolean hiveWantsMysql = HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY)); - boolean topologyContainsMysql = topology.getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT); + boolean topologyContainsMysql = topology.getComponents() + .anyMatch(c -> MYSQL_SERVER_COMPONENT.equals(c.getComponentName()) && HIVE_SERVICE.equals(c.getServiceName())); if (topologyContainsMysql && !hiveWantsMysql) { String errorMessage = String.format( diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java index 1fa043c0eb8..fbfb03da890 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -29,7 +28,6 @@ import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.ClusterTopologyImpl; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.InvalidTopologyException; import org.slf4j.Logger; @@ -49,14 +47,14 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { Map> clusterConfigurations = topology.getConfiguration().getProperties(); - if (!ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations)) { + if (!BlueprintConfigurationProcessor.isNameNodeHAEnabled(clusterConfigurations)) { LOG.info("NAMENODE HA is not enabled, skipping validation of {}", blueprint.getName()); return; } LOG.info("Validating NAMENODE HA for blueprint: {}", blueprint.getName()); - List hostGroupsForComponent = new ArrayList<>(topology.getBlueprint().getHostGroupsForComponent("NAMENODE")); + List hostGroupsForComponent = new ArrayList<>(topology.getHostGroupsForComponent("NAMENODE")); for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { Map> operationalConfiguration = new HashMap<>(clusterConfigurations); @@ -72,13 +70,8 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); } if (BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()) { - for (HostGroup hostGroupForComponent : hostGroupsForComponent) { - Iterator itr = givenHostGroups.iterator(); - while(itr.hasNext()){ - if(itr.next().contains(hostGroupForComponent.getName())){ - itr.remove(); - } - } + for (String hostGroupForComponent : hostGroupsForComponent) { + givenHostGroups.removeIf(s -> s.contains(hostGroupForComponent)); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java index 6a9706f3581..caa3505de41 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java @@ -22,6 +22,7 @@ import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +32,8 @@ public class RejectUnknownComponents implements TopologyValidator { @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { - String unknownComponents = topology.getComponentNames() + String unknownComponents = topology.getComponents() + .map(ResolvedComponent::getComponentName) .filter(c -> !RootComponent.AMBARI_SERVER.name().equals(c)) .filter(c -> !topology.getStack().getComponents().contains(c)) .collect(joining(", ")); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index e79dc407463..7fb80733bfe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -1,5 +1,3 @@ -package org.apache.ambari.server.topology.validators; - /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,32 +12,30 @@ * limitations under the License. */ -import java.util.Collection; +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; + import java.util.HashMap; -import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; /** * Validates that all required passwords are provided. */ public class RequiredPasswordValidator implements TopologyValidator { - // todo remove the field as all the information is available in the topology being validated - private String defaultPassword; - - public RequiredPasswordValidator() { - } - /** * Validate that all required password properties have been set or that 'default_password' is specified. * @@ -48,8 +44,7 @@ public RequiredPasswordValidator() { */ public void validate(ClusterTopology topology) throws InvalidTopologyException { - defaultPassword = topology.getDefaultPassword(); - Map>> missingPasswords = validateRequiredPasswords(topology); + Map>> missingPasswords = validateRequiredPasswords(topology); if (! missingPasswords.isEmpty()) { throw new InvalidTopologyException("Missing required password properties. Specify a value for these " + @@ -67,86 +62,45 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { * * @throws IllegalArgumentException if blueprint contains invalid information */ + private Map>> validateRequiredPasswords(ClusterTopology topology) { + Map>> missingProperties = new HashMap<>(); - //todo: this is copied/pasted from Blueprint and is currently only used by validatePasswordProperties() - //todo: seems that we should have some common place for this code so it can be used by BP and here? - private Map>> validateRequiredPasswords(ClusterTopology topology) { - - Map>> missingProperties = - new HashMap<>(); + StackDefinition stack = topology.getStack(); + String defaultPassword = topology.getDefaultPassword(); + boolean hasDefaultPassword = defaultPassword != null && !defaultPassword.trim().isEmpty(); for (Map.Entry groupEntry: topology.getHostGroupInfo().entrySet()) { String hostGroupName = groupEntry.getKey(); Map> groupProperties = groupEntry.getValue().getConfiguration().getFullProperties(3); - Collection processedServices = new HashSet<>(); - Blueprint blueprint = topology.getBlueprint(); - StackDefinition stack = topology.getStack(); - - HostGroup hostGroup = blueprint.getHostGroup(hostGroupName); - for (String component : hostGroup.getComponentNames()) { - //for now, AMBARI is not recognized as a service in Stacks - if (component.equals(RootComponent.AMBARI_SERVER.name())) { - continue; - } - - String serviceName = stack.getServiceForComponent(component); - if (processedServices.add(serviceName)) { - //todo: do I need to subtract excluded configs? - Collection requiredProperties = - stack.getRequiredConfigurationProperties(serviceName, PropertyInfo.PropertyType.PASSWORD); - - for (Stack.ConfigProperty property : requiredProperties) { - String category = property.getType(); - String name = property.getName(); - if (! propertyExists(topology, groupProperties, category, name)) { - Map> missingHostGroupPropsMap = missingProperties.computeIfAbsent(hostGroupName, __ -> new HashMap<>()); - Collection missingHostGroupTypeProps = missingHostGroupPropsMap.computeIfAbsent(category, __ -> new HashSet<>()); - missingHostGroupTypeProps.add(name); + Map> missingPropertiesInHostGroup = topology.getComponentsInHostGroup(hostGroupName) + .filter(component -> !RootComponent.AMBARI_SERVER.name().equals(component.getComponentName())) + .map(ResolvedComponent::getServiceType) + .distinct() + .flatMap(serviceType -> stack.getRequiredConfigurationProperties(serviceType, PropertyInfo.PropertyType.PASSWORD).stream()) + .filter(property -> !propertyExists(groupProperties, property.getType(), property.getName())) + .collect(groupingBy(Stack.ConfigProperty::getType, mapping(Stack.ConfigProperty::getName, toSet()))); + + if (!missingPropertiesInHostGroup.isEmpty()) { + if (hasDefaultPassword) { + for (Map.Entry> entry : missingPropertiesInHostGroup.entrySet()) { + String type = entry.getKey(); + for (String name : entry.getValue()) { + topology.getConfiguration().setProperty(type, name, defaultPassword); } } + } else { + missingProperties.put(hostGroupName, missingPropertiesInHostGroup); } } } return missingProperties; } - private boolean propertyExists(ClusterTopology topology, Map> props, String type, String property) { + private boolean propertyExists(Map> props, String type, String property) { Map typeProps = props.get(type); - return (typeProps != null && typeProps.containsKey(property)) || setDefaultPassword(topology, type, property); + return typeProps != null && typeProps.containsKey(property); } - /** - * Attempt to set the default password in cluster configuration for missing password property. - * - * @param configType configuration type - * @param property password property name - * - * @return true if password was set, otherwise false. Currently the password will always be set - * unless it is null - */ - private boolean setDefaultPassword(ClusterTopology topology, String configType, String property) { - boolean setDefaultPassword = false; - if (defaultPassword != null && ! defaultPassword.trim().isEmpty()) { - topology.getConfiguration().setProperty(configType, property, defaultPassword); - setDefaultPassword = true; - } - return setDefaultPassword; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - RequiredPasswordValidator that = (RequiredPasswordValidator) o; - - return defaultPassword == null ? that.defaultPassword == null : defaultPassword.equals(that.defaultPassword); - } - - @Override - public int hashCode() { - return defaultPassword != null ? defaultPassword.hashCode() : 0; - } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java index 6642f58976d..1deb685c73a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java @@ -163,7 +163,6 @@ public void setup() throws Exception { groupInfoMap.put("host_group_1", group1Info); groupInfoMap.put("host_group_2", group2Info); - expect(topology.isNameNodeHAEnabled()).andReturn(false).anyTimes(); expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); @@ -219,7 +218,6 @@ private void setupMocksForKerberosEnabledCluster() throws Exception { groupInfoMap.put("host_group_1", group1Info); groupInfoMap.put("host_group_2", group2Info); - expect(topology.isNameNodeHAEnabled()).andReturn(false).anyTimes(); expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 57e39e9ecd2..f4c0f551f58 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1064,9 +1064,9 @@ public void testCreateServiceComponentWithInvalidRequest() try { set1.clear(); ServiceComponentRequest rInvalid1 = - new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", null); + new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", State.INSTALLED.name()); ServiceComponentRequest rInvalid2 = - new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", null); + new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", State.STARTED.name()); set1.add(rInvalid1); set1.add(rInvalid2); ComponentResourceProviderTest.createComponents(controller, set1); @@ -3172,7 +3172,7 @@ public void testServiceUpdateInvalidRequest() throws Exception { req1 = new ServiceRequest(cluster1, serviceGroupName, serviceName1, repositoryVersion02.getId(), State.INSTALLED.toString(), null); req2 = new ServiceRequest(cluster1, serviceGroupName, serviceName1, repositoryVersion02.getId(), - State.INSTALLED.toString(), null); + State.STARTED.toString(), null); reqs.add(req1); reqs.add(req2); ServiceResourceProviderTest.updateServices(controller, reqs, mapRequestProps, true, false); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 99096438b5b..9fbe39eb69c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,6 +18,10 @@ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.toSet; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.ALWAYS_APPLY; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.NEVER_APPLY; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.expect; @@ -57,6 +61,7 @@ import org.apache.ambari.server.topology.AdvisedConfiguration; import org.apache.ambari.server.topology.AmbariContext; import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.BlueprintBasedClusterProvisionRequest; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ClusterTopologyImpl; @@ -67,7 +72,9 @@ import org.apache.ambari.server.topology.HostGroupImpl; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyRequest; +import org.apache.ambari.server.topology.ResolvedComponent; +import org.apache.ambari.server.topology.SecurityConfiguration; +import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.commons.lang.StringUtils; import org.easymock.EasyMock; import org.easymock.EasyMockRule; @@ -138,11 +145,14 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { private Cluster cluster; @Mock - private TopologyRequest topologyRequestMock; + private ProvisionClusterRequest topologyRequestMock; @Mock(type = MockType.NICE) private ConfigHelper configHelper; + @Mock(type = MockType.NICE) + private SecurityConfigurationFactory securityFactory; + @Before public void init() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); @@ -152,19 +162,17 @@ public void init() throws Exception { expect(stack.getName()).andReturn(STACK_NAME).atLeastOnce(); expect(stack.getVersion()).andReturn(STACK_VERSION).atLeastOnce(); // return false for all components since for this test we don't care about the value - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(Collections.emptyMap()).anyTimes(); - expect(serviceInfo.getRequiredProperties()).andReturn( - Collections.emptyMap()).anyTimes(); + expect(serviceInfo.getRequiredProperties()).andReturn(Collections.emptyMap()).anyTimes(); expect(serviceInfo.getRequiredServices()).andReturn(Collections.emptyList()).anyTimes(); setupGetServiceForComponentExpectations(); expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - Set emptySet = Collections.emptySet(); - expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); + expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(Collections.emptySet()).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); expect(configHelper.getDefaultStackProperties( @@ -290,7 +298,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -338,14 +346,14 @@ public void testDoUpdateForBlueprintExport_FilterProperties() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals(properties.size(), 3); - assertEquals(((Map) properties.get("kerberos-env")).size(), 0); - assertEquals(((Map) properties.get("krb5-conf")).size(), 0); - assertEquals(((Map) properties.get("tez-site")).size(), 0); + assertEquals((properties.get("kerberos-env")).size(), 0); + assertEquals((properties.get("krb5-conf")).size(), 0); + assertEquals((properties.get("tez-site")).size(), 0); } @Test @@ -372,7 +380,7 @@ public void testDoUpdateForBlueprintExportRangerHAPolicyMgrExternalUrlProperty() Collection hostGroups = ImmutableSet.of(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -416,7 +424,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty_specifiedInParentC hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -464,7 +472,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty_hostGroupConfigura hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -499,7 +507,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty__withPort() throws hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -532,7 +540,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty__ExternalReference hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -578,7 +586,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty() throws Exception hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -625,7 +633,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty__WithPorts() throws hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -680,7 +688,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty__YAML() throws Exce hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -723,7 +731,7 @@ public void testDoUpdateForBlueprintExport_DBHostProperty() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -756,7 +764,7 @@ public void testDoUpdateForBlueprintExport_DBHostProperty__External() throws Exc hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -808,7 +816,7 @@ public void testDoUpdateForBlueprintExport_PasswordFilterApplied() throws Except hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -891,7 +899,7 @@ public void testFalconConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -928,7 +936,7 @@ public void testTezConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -940,8 +948,6 @@ public void testTezConfigExport() throws Exception { * There is no support currently for deploying a fully Kerberized * cluster with Blueprints. This test verifies the current treatment * of Kerberos-related properties in a Blueprint export. - * - * @throws Exception */ @Test public void testKerberosConfigExport() throws Exception { @@ -979,7 +985,7 @@ public void testKerberosConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1043,7 +1049,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabled() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1116,7 +1122,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledPrimaryNamePreferen Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1176,7 +1182,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledNameServiceProperti Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1212,7 +1218,7 @@ public void testDoNameNodeHighAvailabilityExportWithHANotEnabled() throws Except Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1275,7 +1281,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledMultipleServices() Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1352,7 +1358,7 @@ public void testYarnConfigExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1418,7 +1424,7 @@ public void testYarnConfigExportedWithDefaultZeroHostAddress() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1494,7 +1500,7 @@ public void testHDFSConfigExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1581,7 +1587,7 @@ public void testHiveConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1674,7 +1680,7 @@ public void testHiveConfigExportedMultipleHiveMetaStoreServers() throws Exceptio hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1770,7 +1776,7 @@ public void testOozieConfigExported() throws Exception { BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").remove("oozie.service.JPAService.jdbc.url"); } - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1833,7 +1839,7 @@ public void testOozieJDBCPropertiesNotRemoved() throws Exception { expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); @@ -1872,7 +1878,7 @@ public void testOozieJDBCPropertyAddedToSingleHostMapDuringImport() throws Excep expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); @@ -1935,7 +1941,7 @@ public void testZookeeperConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2015,7 +2021,7 @@ public void testKnoxSecurityConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2064,7 +2070,7 @@ public void testKafkaConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2103,7 +2109,7 @@ public void testPropertyWithUndefinedHostisExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2145,7 +2151,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue() thro hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set configTypesUpdated = @@ -2201,7 +2207,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_provid hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2248,7 +2254,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_hostGr hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2296,7 +2302,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_BPHost hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); // todo: set as BP hostgroup topology.getHostGroupInfo().get("group2").getConfiguration().setParentConfiguration(group2BPConfig); @@ -2335,7 +2341,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MissingComponent() expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); //todo: should throw a checked exception, not the exception expected by the api @@ -2378,7 +2384,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleMatchingHos expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); try { @@ -2422,7 +2428,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleAppTimeline expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2458,7 +2464,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MissingOptionalComp expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2490,7 +2496,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue__WithP hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("core-site").get("fs.defaultFS"); @@ -2535,7 +2541,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues() thro hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-site").get("hbase.zookeeper.quorum"); @@ -2592,7 +2598,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___with hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("webhcat-site").get("templeton.zookeeper.hosts"); @@ -2643,7 +2649,7 @@ public void testMultipleHostTopologyUpdater__localhost__singleHost() throws Exce hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2677,7 +2683,7 @@ public void testMultipleHostTopologyUpdater__localhost__singleHostGroup() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2712,7 +2718,7 @@ public void testMultipleHostTopologyUpdater__hostgroup__singleHostGroup() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2732,7 +2738,7 @@ public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); @@ -2772,7 +2778,7 @@ public void testDoUpdateForClusterVerifyRetrySettingsCustomized() throws Excepti Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); @@ -2860,7 +2866,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabledSpecifyingHostNamesDirect expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2935,7 +2941,7 @@ public void testHiveConfigClusterUpdateCustomValueSpecifyingHostNamesMetaStoreHA hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2984,7 +2990,7 @@ public void testHiveConfigClusterUpdateSpecifyingHostNamesHiveServer2HA() throws expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3048,7 +3054,7 @@ public void testHiveConfigClusterUpdateUsingExportedNamesHiveServer2HA() throws expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3109,7 +3115,7 @@ private void testHiveMetastoreHA(String separator) throws InvalidTopologyExcepti hostGroups.add(new TestHostGroup("host_group_" + i, components, Collections.singleton(hostNames[i]))); } - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedValue = webHCatSiteProperties.get(propertyKey); @@ -3163,7 +3169,7 @@ public void testHiveInteractiveLlapZookeeperConfigExported() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3217,7 +3223,7 @@ public void testOozieConfigClusterUpdateHAEnabledSpecifyingHostNamesDirectly() t expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3283,7 +3289,7 @@ public void testOozieHAEnabledExport() throws Exception { expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3337,7 +3343,7 @@ public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirect expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3424,7 +3430,7 @@ public void testYarnHighAvailabilityExport() throws Exception { expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3507,7 +3513,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURLSpecifyingHostNamesDirect hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3556,7 +3562,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___YAML hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3607,7 +3613,7 @@ public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__defaultValues_ hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3657,7 +3663,7 @@ public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__FQDN_ValuesSpe hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3692,7 +3698,7 @@ public void testDoUpdateForClusterCreate_MProperty__defaultValues() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3724,7 +3730,7 @@ public void testDoUpdateForClusterCreate_MProperty__missingM() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3756,7 +3762,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue() thr hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3788,7 +3794,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_Using hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3820,7 +3826,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_WithP hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3852,7 +3858,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue__With hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3898,7 +3904,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues() thr hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3956,7 +3962,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___wit hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4014,7 +4020,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___wit hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4055,7 +4061,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPo Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4111,7 +4117,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___YAM hostGroups.add(group3); hostGroups.add(group4); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4185,7 +4191,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__defaultValue() throws E hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4221,7 +4227,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__exportedValue() throws hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4254,7 +4260,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__external() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4269,7 +4275,7 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")); @@ -4297,7 +4303,7 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4313,7 +4319,7 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); @@ -4338,7 +4344,7 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4353,7 +4359,7 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(Collections.emptyMap()).anyTimes(); // customized stack calls for this test only @@ -4380,7 +4386,7 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4396,7 +4402,7 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only Set excludedConfigTypes = new HashSet<>(); @@ -4429,7 +4435,7 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4467,7 +4473,7 @@ public void testFalconConfigClusterUpdate() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4508,7 +4514,7 @@ public void testFalconConfigClusterUpdateDefaultConfig() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -4544,7 +4550,7 @@ public void testHiveConfigClusterUpdateCustomValue() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4600,7 +4606,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4622,7 +4628,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4679,7 +4685,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4701,7 +4707,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4759,7 +4765,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4780,7 +4786,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4847,7 +4853,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4869,7 +4875,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4917,7 +4923,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4938,7 +4944,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4976,7 +4982,7 @@ public void testHiveConfigClusterUpdateDefaultValue() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5029,7 +5035,7 @@ public void testAtlas() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5093,7 +5099,7 @@ public void testHiveConfigClusterUpdateExportedHostGroupValue() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5133,7 +5139,7 @@ public void testStormAndKafkaConfigClusterUpdateWithoutGangliaServer() throws Ex expect(stack.getCardinality("GANGLIA_SERVER")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5181,7 +5187,7 @@ public void testStormandKafkaConfigClusterUpdateWithGangliaServer() throws Excep Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5276,7 +5282,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabled() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set updatedConfigTypes = @@ -5395,7 +5401,7 @@ public void testDoUpdateForClusterWithNameNodeHANotEnabled() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set updatedConfigTypes = @@ -5464,7 +5470,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabledAndActiveNodeSet() throws Collection hostGroups = new ArrayList<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -5611,7 +5617,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURL() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5653,7 +5659,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURL_UsingMinusSymbolInHostNa hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5704,7 +5710,7 @@ public void testHadoopHaNameNode() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -5744,7 +5750,7 @@ public void testGetRequiredHostGroups___validComponentCountOfZero() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5777,7 +5783,7 @@ public void testGetRequiredHostGroups___invalidComponentCountOfZero() throws Exc hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5812,7 +5818,7 @@ public void testGetRequiredHostGroups___multipleGroups() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5856,7 +5862,7 @@ public void testAllDefaultUserAndGroupProxyPropertiesSet() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5901,7 +5907,7 @@ public void testRelevantDefaultUserAndGroupProxyPropertiesSet() throws Exception Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5945,7 +5951,7 @@ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided() throws Ex Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5988,7 +5994,7 @@ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided2() throws E Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6032,7 +6038,7 @@ public void testHiveWithoutAtlas() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6123,7 +6129,7 @@ private void validateAtlasHivePropertiesForTestCase(Map hostGroups = Collections.singletonList(group1); - ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology1); configProcessor.doUpdateForClusterCreate(); @@ -6171,7 +6177,7 @@ public void testAtlasHivePropertiesWithHTTPS() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6203,7 +6209,7 @@ public void testStormAmsPropertiesDefault() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6234,7 +6240,7 @@ public void testStormAmsPropertiesUserDefinedReporter() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6265,7 +6271,7 @@ public void testKafkaAmsProperties() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6297,7 +6303,7 @@ public void testKafkaAmsPropertiesMultipleReporters() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6347,9 +6353,8 @@ public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ONLY_STACK_DEFAULTS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); @@ -6407,9 +6412,8 @@ public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ONLY_STACK_DEFAULTS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); @@ -6476,9 +6480,8 @@ public void testRecommendConfiguration_applyAlways() throws Exception { Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ALWAYS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ALWAYS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // WHEN Set configTypes = configProcessor.doUpdateForClusterCreate(); @@ -6531,9 +6534,8 @@ public void testRecommendConfiguration_neverApply() throws Exception { Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // WHEN configProcessor.doUpdateForClusterCreate(); @@ -6572,7 +6574,7 @@ public void testRangerAdminProperties() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6608,7 +6610,7 @@ public void testRangerAdminProperties_defaults() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6645,7 +6647,7 @@ public void testRangerAdminProperties_HA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6700,7 +6702,7 @@ public void testRangerEnv_defaults() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6759,7 +6761,7 @@ public void testRangerEnv_defaults_NO_HDFS() throws Exception { Collection hostGroups = Lists.newArrayList(group1);//, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6825,7 +6827,7 @@ public void testRangerEnv() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6899,7 +6901,7 @@ public void testRangerEnvWithHdfsHA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6965,7 +6967,7 @@ public void testRangerEnvBlueprintExport() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7037,7 +7039,7 @@ public void testRangerEnvExportBlueprintWithHdfsHA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7088,7 +7090,7 @@ public void testRangerKmsServerProperties() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7128,7 +7130,7 @@ public void testRangerKmsServerProperties_default() throws Exception { Collection hostGroups = Collections.singleton(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7173,7 +7175,7 @@ public void testHdfsWithRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7227,7 +7229,7 @@ public void testHdfsWithNoRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); @@ -7272,7 +7274,7 @@ public void testHdfsWithRangerKmsServer_default() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7318,7 +7320,7 @@ public void testHdfsWithRangerKmsServer__multiple_hosts__localhost() throws Exce Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7372,7 +7374,7 @@ public void testHdfsWithRangerKmsServer__multiple_hosts__hostgroup() throws Exce Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7428,7 +7430,7 @@ public void testResolutionOfDRPCServerAndNN() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7472,7 +7474,7 @@ public void testHadoopWithRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7516,7 +7518,7 @@ public void testHadoopWithNoRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); @@ -7561,7 +7563,7 @@ public void testHadoopWithRangerKmsServer_default() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7816,7 +7818,7 @@ public void testHawqConfigClusterUpdate() throws Exception { hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -7848,7 +7850,7 @@ public void testHawqNonHaConfigClusterUpdate() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -7871,7 +7873,7 @@ public void testDoUpdateForBlueprintExport_NonTopologyProperty__AtlasClusterName Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); Long clusterId = topology.getClusterId(); Map typeProps = new HashMap<>(); typeProps.put("atlas.cluster.name", String.valueOf(clusterId)); @@ -7901,8 +7903,7 @@ public void testDoUpdateForBlueprintExport_NonTopologyProperty() throws Exceptio Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - Long clusterId = topology.getClusterId(); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); Map hiveSiteProps = new HashMap<>(); hiveSiteProps.put("hive.exec.post.hooks", someString); @@ -7939,8 +7940,8 @@ public void druidProperties() throws Exception { properties.put("druid-common", druidCommon); Map> parentProperties = new HashMap<>(); - Configuration parentClusterConfig = new Configuration(parentProperties, Collections.>>emptyMap()); - Configuration clusterConfig = new Configuration(properties, Collections.>>emptyMap(), parentClusterConfig); + Configuration parentClusterConfig = new Configuration(parentProperties, Collections.emptyMap()); + Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); Collection hgComponents1 = Sets.newHashSet("DRUID_COORDINATOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); @@ -7950,7 +7951,7 @@ public void druidProperties() throws Exception { Collection hostGroups = Arrays.asList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -7980,7 +7981,7 @@ public void testAmsPropertiesDefault() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8010,7 +8011,7 @@ public void testAmsPropertiesSpecialAddress() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8045,7 +8046,7 @@ public void testAmsPropertiesSpecialAddressMultipleCollectors() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8080,8 +8081,8 @@ public void testStackPasswordPropertyFilter() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - expect(stack.isPasswordProperty((String) anyObject(), (String) anyObject(), (String) anyObject())).andReturn(true).once(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + expect(stack.isPasswordProperty(anyObject(), anyObject(), anyObject())).andReturn(true).once(); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -8160,7 +8161,7 @@ public void testValuesTrimming() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8213,7 +8214,7 @@ private Configuration createStackDefaults() { } private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration configuration, - Collection hostGroups) + Collection hostGroups, ConfigRecommendationStrategy recommendation) throws InvalidTopologyException { @@ -8221,6 +8222,7 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration Map hostGroupInfo = new HashMap<>(); Map allHostGroups = new HashMap<>(); + Map> resolvedComponents = new HashMap<>(); for (TestHostGroup hostGroup : hostGroups) { HostGroupInfo groupInfo = new HostGroupInfo(hostGroup.name); @@ -8232,11 +8234,14 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration for (String componentName : hostGroup.components) { componentList.add(new Component(componentName, null, stack.getServiceForComponent(componentName), null)); } + Set components = componentList.stream() + .map(comp -> new ResolvedComponent(STACK_ID, comp.getServiceInstance(), comp.getServiceInstance(), comp)) + .collect(toSet()); //create host group which is set on topology allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, componentList, EMPTY_CONFIG, "1")); - hostGroupInfo.put(hostGroup.name, groupInfo); + resolvedComponents.put(hostGroup.name, components); } for (HostGroup group : allHostGroups.values()) { @@ -8245,17 +8250,27 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration expect(bp.getHostGroups()).andReturn(allHostGroups).anyTimes(); - expect(topologyRequestMock.getClusterId()).andReturn(1L).anyTimes(); expect(topologyRequestMock.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topologyRequestMock.getClusterId()).andReturn(1L).anyTimes(); + expect(topologyRequestMock.getConfigRecommendationStrategy()).andReturn(recommendation).anyTimes(); expect(topologyRequestMock.getConfiguration()).andReturn(configuration).anyTimes(); + expect(topologyRequestMock.getDefaultPassword()).andReturn("secret").anyTimes(); expect(topologyRequestMock.getHostGroupInfo()).andReturn(hostGroupInfo).anyTimes(); - - replay(bp, topologyRequestMock); - - ClusterTopology topology = new ClusterTopologyImpl(ambariContext, topologyRequestMock); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); - - return topology; + expect(topologyRequestMock.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(topologyRequestMock.getProvisionAction()).andReturn(ProvisionAction.INSTALL_AND_START).anyTimes(); + expect(topologyRequestMock.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(topologyRequestMock.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(bp.getConfiguration()).andReturn(Configuration.createEmpty()).anyTimes(); + expect(bp.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(bp.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(securityFactory.loadSecurityConfigurationByReference(anyString())).andReturn(null).anyTimes(); + replay(bp, topologyRequestMock, securityFactory); + + BlueprintBasedClusterProvisionRequest request = new BlueprintBasedClusterProvisionRequest(ambariContext, securityFactory, bp, topologyRequestMock); + + ClusterTopologyImpl clusterTopology = new ClusterTopologyImpl(ambariContext, request, resolvedComponents); + clusterTopology.setClusterId(1L); + return clusterTopology; } private class TestHostGroup { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index c0ce6b322b9..66893f38222 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -240,16 +240,16 @@ public void setUp() throws Exception { expect(topology.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getBlueprintName()).andReturn(BP_NAME).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.singletonMap(HOST_GROUP_1, group1Info)).anyTimes(); expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); expect(topology.getServices()).andReturn(blueprintServices).anyTimes(); - expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); - expect(topology.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); - expect(topology.getStackIdsForService("service1")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(topology.getStackIdsForService("service2")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for both services + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index ac96d1912af..9bab7c620cd 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -38,6 +38,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor; @@ -260,9 +261,9 @@ private Capture> testProcessWithKerberos(String blueprintP List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(topology.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(topology.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(topology.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for all services + )).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); @@ -316,7 +317,7 @@ private Capture> testProcessWithKerberos(String blueprintP } @Test - public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Exception { + public void testProcessClusterConfigRequestDontIncludeKerberosConfigs() throws Exception { Map> existingConfig = new HashMap<>(); Configuration stackConfig = new Configuration(existingConfig, @@ -350,11 +351,10 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(topology.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(topology.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(topology.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); - expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for all services + )).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 618b00192d3..3e02d2c66a8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -40,6 +40,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -258,8 +259,9 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(topology.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for both services + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -272,7 +274,6 @@ public void setup() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() @@ -323,7 +324,7 @@ public void setup() throws Exception { expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); @@ -331,7 +332,7 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index a15318eb4fa..772fd14d056 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -255,8 +256,9 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes(); - expect(topology.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for both services + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -269,7 +271,6 @@ public void setup() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() @@ -320,7 +321,7 @@ public void setup() throws Exception { expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Arrays.asList("component1")) @@ -332,7 +333,7 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index 8e03c33a13c..2f2cab89aac 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -258,8 +259,9 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(topology.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for both services + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -272,7 +274,6 @@ public void setup() throws Exception { expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() @@ -323,7 +324,7 @@ public void setup() throws Exception { expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); @@ -332,7 +333,7 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java similarity index 86% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java index 484ade0bf6d..7ab4e4762e8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownComponentsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ambari.server.topology.validators; +package org.apache.ambari.server.topology; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; @@ -25,17 +25,17 @@ import java.util.stream.Stream; import org.apache.ambari.server.controller.internal.StackDefinition; -import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.validators.RejectUnknownComponents; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.junit.After; import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableSet; -public class RejectUnknownComponentsTest { +public class ComponentResolverTest { - private TopologyValidator validator = new RejectUnknownComponents(); + private final TopologyValidator validator = new RejectUnknownComponents(); private final ClusterTopology topology = createNiceMock(ClusterTopology.class); private final StackDefinition stack = createNiceMock(StackDefinition.class); @@ -73,7 +73,7 @@ public void rejectsUnknownComponents() throws Exception { } private void componentsInTopologyAre(String... components) { - expect(topology.getComponentNames()).andReturn(Stream.of(components)).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.empty()).anyTimes(); // FIXME replay(topology); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java index 0bb725320a0..6df1559af98 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java @@ -199,7 +199,7 @@ public void testPersistedRequestsWithReservedHosts() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -302,7 +302,7 @@ public void testPersistedRequestsWithHostPredicate() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_2"))).andReturn(hostGroup2).atLeastOnce(); - expect(clusterTopology.containsMasterComponent(hostGroup2)).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_2")).andReturn(false).atLeastOnce(); replayAll(); @@ -413,7 +413,7 @@ public void testRemoveHostRequestByHostName() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -511,7 +511,7 @@ public void testRemovePendingHostRequests() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -597,7 +597,7 @@ private LogicalRequest createTopologyRequestByHostCount(int hostCount, String ho expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).anyTimes(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(hostRequests).anyTimes(); expect(blueprint.getHostGroup(eq(hostGroupEntity.getName()))).andReturn(hostGroup1).anyTimes(); - expect(clusterTopology.containsMasterComponent(hostGroup1)).andReturn(false).anyTimes(); + expect(clusterTopology.containsMasterComponent(hostGroupName)).andReturn(false).anyTimes(); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index b50c3ca1604..49e2023550e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -47,6 +47,7 @@ import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleStatus; @@ -261,8 +262,9 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(clusterTopologyMock.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(clusterTopologyMock.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(clusterTopologyMock.getComponents()).andReturn(Stream.of( + // FIXME add ResolvedComponents for both services + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -274,7 +276,6 @@ public void setup() throws Exception { expect(clusterTopologyMock.getStack()).andReturn(stack).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); // don't expect toEntity() @@ -315,14 +316,14 @@ public void setup() throws Exception { expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(clusterTopologyMock.containsMasterComponent(group1)).andReturn(true).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(clusterTopologyMock.containsMasterComponent(group2)).andReturn(false).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index ee893185b09..12ca8e3f065 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -18,6 +18,7 @@ import static org.easymock.EasyMock.replay; import java.util.Arrays; +import java.util.stream.Stream; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.Configuration; @@ -132,7 +133,7 @@ private void topologyHasMysql(boolean hasMysql) { if (hasMysql) { components.add("MYSQL_SERVER"); } - expect(topology.getComponentNames("HIVE")).andReturn(components.build()).anyTimes(); + expect(topology.getComponents()).andReturn(Stream.empty()).anyTimes(); // FIXME expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); replay(topology); } From 303662c1e3a67c6d32e821c90e1eb367791845e1 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 14 Feb 2018 11:22:47 +0100 Subject: [PATCH 17/25] Add Freebuilder --- ambari-project/pom.xml | 7 +++++++ ambari-server/pom.xml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml index 7bb01b5360c..ff338e85a35 100644 --- a/ambari-project/pom.xml +++ b/ambari-project/pom.xml @@ -485,6 +485,13 @@ swagger-maven-plugin ${swagger.maven.plugin.version} + + org.inferred + freebuilder + 1.14.8 + true + provided + diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 7ceae4872c3..31d97ed050a 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -1644,6 +1644,10 @@ 0.1.10 test + + org.inferred + freebuilder + From e3fbb9d694150b909e9ae47927b45021f52e886d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 15 Feb 2018 13:43:39 +0100 Subject: [PATCH 18/25] Use FreeBuilder for ResolvedComponent --- .../server/controller/ControllerModule.java | 3 + .../ambari/server/topology/AmbariContext.java | 12 +- .../topology/ClusterConfigurationRequest.java | 5 +- .../server/topology/ClusterTopologyImpl.java | 10 +- .../server/topology/ComponentResolver.java | 142 +---------------- .../server/topology/ResolvedComponent.java | 72 +++++---- .../topology/StackComponentResolver.java | 144 ++++++++++++++++++ .../server/topology/TopologyManager.java | 5 +- .../validators/HiveServiceValidator.java | 2 +- .../validators/RejectUnknownComponents.java | 2 +- .../validators/RequiredPasswordValidator.java | 4 +- .../BlueprintConfigurationProcessorTest.java | 13 +- .../server/topology/AmbariContextTest.java | 14 +- .../ClusterConfigurationRequestTest.java | 46 ++---- .../ClusterDeployWithStartOnlyTest.java | 30 ++-- ...stallWithoutStartOnComponentLevelTest.java | 31 ++-- .../ClusterInstallWithoutStartTest.java | 39 +++-- .../server/topology/LogicalRequestTest.java | 3 +- ...t.java => StackComponentResolverTest.java} | 6 +- .../server/topology/TopologyManagerTest.java | 21 ++- ...DependencyAndCardinalityValidatorTest.java | 2 +- .../validators/HiveServiceValidatorTest.java | 7 +- .../RequiredPasswordValidatorTest.java | 28 ++-- 23 files changed, 339 insertions(+), 302 deletions(-) create mode 100644 ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java rename ambari-server/src/test/java/org/apache/ambari/server/topology/{ComponentResolverTest.java => StackComponentResolverTest.java} (92%) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index 1adc8474ab6..0ae2676ee5a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -169,10 +169,12 @@ import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl; import org.apache.ambari.server.topology.BlueprintFactory; +import org.apache.ambari.server.topology.ComponentResolver; import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; import org.apache.ambari.server.topology.PersistedStateImpl; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.StackComponentResolver; import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; @@ -416,6 +418,7 @@ protected void configure() { bind(SecurityConfigurationFactory.class).in(Scopes.SINGLETON); bind(PersistedState.class).to(PersistedStateImpl.class); + bind(ComponentResolver.class).to(StackComponentResolver.class); // factory to create LoggingRequestHelper instances for LogSearch integration bind(LoggingRequestHelperFactory.class).to(LoggingRequestHelperFactoryImpl.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index c94495a165b..4b0ace6a4d7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -350,20 +350,20 @@ private void createAmbariClusterResource(String clusterName, Set stackI private void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { Set serviceGroupRequests = topology.getComponents() - .map(c -> new ServiceGroupRequest(clusterName, c.getServiceGroupName())) + .map(c -> new ServiceGroupRequest(clusterName, c.effectiveServiceGroupName())) .collect(toSet()); Set serviceRequests = topology.getComponents() .map(c -> new ServiceRequest( - clusterName, c.getServiceGroupName(), c.getServiceName(), c.getServiceType(), repoVersionByStack.get(c.getStackId()), null, - topology.getSetting().getCredentialStoreEnabled(c.getServiceName()), // FIXME settings by service type or name? - c.getStackId() + clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.serviceType(), repoVersionByStack.get(c.stackId()), null, + topology.getSetting().getCredentialStoreEnabled(c.effectiveServiceName()), // FIXME settings by service type or name? + c.stackId() )) .collect(toSet()); Set componentRequests = topology.getComponents() - .map(c -> new ServiceComponentRequest(clusterName, c.getServiceGroupName(), c.getServiceName(), c.getComponentName(), null, - topology.getSetting().getRecoveryEnabled(c.getServiceName(), c.getComponentName()))) // FIXME settings by service type or name? + .map(c -> new ServiceComponentRequest(clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.componentName(), null, + topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName()))) // FIXME settings by service type or name? .collect(toSet()); try { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 54e28e38806..ae4fe6b6706 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -233,7 +233,8 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> createServiceComponentMap() { return clusterTopology.getComponents() - .collect(groupingBy(ResolvedComponent::getServiceName, mapping(ResolvedComponent::getComponentName, toSet()))); + .collect(groupingBy(ResolvedComponent::effectiveServiceName, + mapping(ResolvedComponent::componentName, toSet()))); } /** @@ -269,7 +270,7 @@ private boolean propertyHasCustomValue(Map clusterConfigProperti private Map createComponentHostMap() { Map componentHostsMap = new HashMap<>(); for (ResolvedComponent component : clusterTopology.getComponents().collect(toSet())) { - String componentName = component.getComponentName(); + String componentName = component.componentName(); Collection componentHost = clusterTopology.getHostAssignmentsForComponent(componentName); // retrieve corresponding clusterInfoKey for component using StageUtils String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(componentName); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index e9e9e26b33b..09d614d3b9f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -31,6 +31,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.BaseClusterRequest; @@ -164,7 +166,7 @@ public Collection getHostGroups() { @Override public Collection getHostGroupsForComponent(String component) { return resolvedComponents.entrySet().stream() - .filter(e -> e.getValue().stream().anyMatch(c -> component.equals(c.getComponentName()))) + .filter(e -> e.getValue().stream().anyMatch(c -> component.equals(c.componentName()))) .map(Map.Entry::getKey) .collect(toSet()); } @@ -229,7 +231,7 @@ public Collection getHostAssignmentsForComponent(String component) { @Override public Collection getServices() { return getComponents() - .map(ResolvedComponent::getServiceName) + .map(ResolvedComponent::effectiveServiceName) .collect(toSet()); } @@ -239,7 +241,7 @@ public Stream getComponents() { .flatMap(Collection::stream); } - @Override + @Override @Nonnull public Stream getComponentsInHostGroup(String hostGroup) { return resolvedComponents.computeIfAbsent(hostGroup, __ -> ImmutableSet.of()).stream(); } @@ -247,7 +249,7 @@ public Stream getComponentsInHostGroup(String hostGroup) { @Override public boolean containsMasterComponent(String hostGroup) { return resolvedComponents.getOrDefault(hostGroup, ImmutableSet.of()).stream() - .anyMatch(ResolvedComponent::isMasterComponent); + .anyMatch(ResolvedComponent::masterComponent); } @Override diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java index 9aefa91fce9..ac74768b35c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java @@ -1,148 +1,10 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.ambari.server.topology; -import static java.util.stream.Collectors.toSet; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Stream; - -import org.apache.ambari.server.controller.internal.StackDefinition; -import org.apache.ambari.server.state.StackId; -import org.apache.commons.lang3.tuple.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Joiner; -import com.google.common.base.Strings; - -/** - * Resolves all incompletely specified host group components in the topology: - * finds stack and/or service type that each component is defined in. - */ -public class ComponentResolver { - - private static final Logger LOG = LoggerFactory.getLogger(ComponentResolver.class); - - private final BlueprintBasedClusterProvisionRequest request; - private final Map uniqueServices; - private final Map> mpackServices; - - public ComponentResolver(BlueprintBasedClusterProvisionRequest request) { - this.request = request; - uniqueServices = request.getUniqueServices(); - mpackServices = request.getServicesByMpack(); - } - - public Map> resolve() { - Map> result = new HashMap<>(); - List problems = new LinkedList<>(); - - StackDefinition stack = request.getStack(); - for (HostGroup hg : request.getHostGroups().values()) { - result.put(hg.getName(), new HashSet<>()); - - for (Component comp : hg.getComponents()) { - Stream> servicesForComponent = stack.getServicesForComponent(comp.getName()); - servicesForComponent = filterByMpackName(comp, servicesForComponent); - servicesForComponent = filterByServiceName(comp, servicesForComponent); - - Set> serviceMatches = servicesForComponent.collect(toSet()); - - if (serviceMatches.size() != 1) { - String msg = formatResolutionProblemMessage(hg, comp, serviceMatches); - LOG.warn("Component resolution failure:" + msg); - problems.add(msg); - } else { - Pair stackService = serviceMatches.iterator().next(); - StackId stackId = stackService.getLeft(); - String serviceType = stackService.getRight(); - String serviceName = comp.getServiceInstance(); - if (Strings.isNullOrEmpty(serviceName)) { - serviceName = serviceType; - } - - ResolvedComponent resolved = new ResolvedComponent(stackId, serviceName, serviceType, comp); - LOG.debug("Component resolved: " + resolved); - result.get(hg.getName()).add(resolved); - } - } - } - - if (!problems.isEmpty()) { - throw new IllegalArgumentException("Component resolution failure:\n" + Joiner.on("\n").join(problems)); - } - - return result; - } - - private static String formatResolutionProblemMessage(HostGroup hg, Component comp, Set> serviceMatches) { - boolean multipleMatches = !serviceMatches.isEmpty(); - String problem = multipleMatches ? "Multiple services" : "No service"; - - StringBuilder sb = new StringBuilder(problem) - .append(" found for component ").append(comp.getName()) - .append(" in host group " ).append(hg.getName()); - - if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { - sb.append(" mpack: ").append(comp.getMpackInstance()); - } - if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { - sb.append(" service: ").append(comp.getServiceInstance()); - } - if (multipleMatches) { - sb.append(": ").append(serviceMatches); - } - - return sb.toString(); - } - - // if component references a specific service instance, filter the stream by the type of that service - private Stream> filterByServiceName(Component comp, Stream> stream) { - if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { - String mpackName = comp.getMpackInstance(); - Map services = !Strings.isNullOrEmpty(mpackName) - ? mpackServices.get(mpackName) - : uniqueServices; - - ServiceInstance service = services.get(comp.getServiceInstance()); - if (service != null) { - String serviceType = service.getType(); - - return stream.filter(pair -> pair.getRight().equals(serviceType)); - } - } - - return stream; - } - // if component references a specific mpack instance, filter the stream by the name of that mpack - private Stream> filterByMpackName(Component comp, Stream> stream) { - if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { - return stream.filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); - } +public interface ComponentResolver { - return stream; - } + Map> resolveComponents(BlueprintBasedClusterProvisionRequest request); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java index ceeacd48196..c4d62a10393 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -17,47 +17,59 @@ */ package org.apache.ambari.server.topology; -import org.apache.ambari.server.state.StackId; - -public class ResolvedComponent { +import java.util.Optional; - private final StackId stackId; - private final String serviceName; - private final String serviceType; - private final Component component; +import org.apache.ambari.server.state.StackId; +import org.inferred.freebuilder.FreeBuilder; - public ResolvedComponent(StackId stackId, String serviceName, String serviceType, Component component) { - this.stackId = stackId; - this.serviceName = serviceName; - this.serviceType = serviceType; - this.component = component; - } +/** + * I provide additional information for a component specified in the blueprint, + * based on values resolved from the stack and sensible defaults. + */ +@FreeBuilder +public interface ResolvedComponent { - public StackId getStackId() { - return stackId; - } + StackId stackId(); + Optional serviceGroupName(); + String serviceType(); + Optional serviceName(); + String componentName(); + boolean masterComponent(); - public String getServiceName() { - return serviceName; - } + /** + * @return the component as specified in the blueprint + */ + Component component(); - public String getServiceType() { - return serviceType; + /** + * @return service group name if it set, otherwise defaults to the stack name + */ + default String effectiveServiceGroupName() { + return serviceGroupName().orElse(stackId().getStackName()); } - public String getComponentName() { - return component.getName(); + /** + * @return service name if it set, otherwise defaults to the service type (eg. ZOOKEEPER) + */ + default String effectiveServiceName() { + return serviceName().orElse(serviceType()); } - public Component getComponent() { - return component; + /** + * Starts building a {@code ResolvedComponent} for the given component. + */ + static Builder builder(Component component) { + return new Builder() + .component(component) + .componentName(component.getName()) + .serviceName(Optional.ofNullable(component.getServiceInstance())) + .serviceGroupName(Optional.ofNullable(component.getMpackInstance())); } - public String getServiceGroupName() { - return stackId.toString(); // FIXME + class Builder extends ResolvedComponent_Builder { + protected Builder() { + masterComponent(false); + } } - public boolean isMasterComponent() { - return false; // FIXME - } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java new file mode 100644 index 00000000000..661ebc8db72 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static java.util.stream.Collectors.toSet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.StackId; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; + +/** + * Resolves all incompletely specified host group components in the topology: + * finds stack and/or service type that each component is defined in. + */ +public class StackComponentResolver implements ComponentResolver { + + private static final Logger LOG = LoggerFactory.getLogger(StackComponentResolver.class); + + @Override + public Map> resolveComponents(BlueprintBasedClusterProvisionRequest request) { + Map uniqueServices = request.getUniqueServices(); + Map> mpackServices = request.getServicesByMpack(); + + Map> result = new HashMap<>(); + List problems = new LinkedList<>(); + + StackDefinition stack = request.getStack(); + for (HostGroup hg : request.getHostGroups().values()) { + result.put(hg.getName(), new HashSet<>()); + + for (Component comp : hg.getComponents()) { + Stream> servicesForComponent = stack.getServicesForComponent(comp.getName()); + servicesForComponent = filterByMpackName(comp, servicesForComponent); + servicesForComponent = filterByServiceName(comp, servicesForComponent, mpackServices, uniqueServices); + + Set> serviceMatches = servicesForComponent.collect(toSet()); + + if (serviceMatches.size() != 1) { + String msg = formatResolutionProblemMessage(hg, comp, serviceMatches); + LOG.warn("Component resolution failure:" + msg); + problems.add(msg); + } else { + Pair stackService = serviceMatches.iterator().next(); + StackId stackId = stackService.getLeft(); + String serviceType = stackService.getRight(); + + ResolvedComponent resolved = ResolvedComponent.builder(comp) + .stackId(stackId) + .serviceType(serviceType) + .build(); + + LOG.debug("Component resolved: " + resolved); + result.get(hg.getName()).add(resolved); + } + } + } + + if (!problems.isEmpty()) { + throw new IllegalArgumentException("Component resolution failure:\n" + Joiner.on("\n").join(problems)); + } + + return result; + } + + private static String formatResolutionProblemMessage(HostGroup hg, Component comp, Set> serviceMatches) { + boolean multipleMatches = !serviceMatches.isEmpty(); + String problem = multipleMatches ? "Multiple services" : "No service"; + + StringBuilder sb = new StringBuilder(problem) + .append(" found for component ").append(comp.getName()) + .append(" in host group " ).append(hg.getName()); + + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + sb.append(" mpack: ").append(comp.getMpackInstance()); + } + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + sb.append(" service: ").append(comp.getServiceInstance()); + } + if (multipleMatches) { + sb.append(": ").append(serviceMatches); + } + + return sb.toString(); + } + + // if component references a specific service instance, filter the stream by the type of that service + private static Stream> filterByServiceName(Component comp, Stream> stream, + Map> mpackServices, Map uniqueServices + ) { + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + String mpackName = comp.getMpackInstance(); + Map services = !Strings.isNullOrEmpty(mpackName) + ? mpackServices.get(mpackName) + : uniqueServices; + + ServiceInstance service = services.get(comp.getServiceInstance()); + if (service != null) { + String serviceType = service.getType(); + + return stream.filter(pair -> pair.getRight().equals(serviceType)); + } + } + + return stream; + } + + // if component references a specific mpack instance, filter the stream by the name of that mpack + private static Stream> filterByMpackName(Component comp, Stream> stream) { + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + return stream.filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); + } + + return stream; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 1ce8131d37b..81082aacf1c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -158,6 +158,9 @@ public class TopologyManager { @Inject private TopologyValidatorService topologyValidatorService; + @Inject + private ComponentResolver resolver; + /** * A boolean not cached thread-local (volatile) to prevent double-checked * locking on the synchronized keyword. @@ -278,7 +281,7 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ ensureInitialized(); BlueprintBasedClusterProvisionRequest provisionRequest = new BlueprintBasedClusterProvisionRequest(ambariContext, securityConfigurationFactory, request.getBlueprint(), request); - Map> resolved = new ComponentResolver(provisionRequest).resolve(); + Map> resolved = resolver.resolveComponents(provisionRequest); final ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, provisionRequest, resolved); final String clusterName = request.getClusterName(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index da6f8044a48..6ba9a2faf8f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -52,7 +52,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { boolean hiveWantsMysql = HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY)); boolean topologyContainsMysql = topology.getComponents() - .anyMatch(c -> MYSQL_SERVER_COMPONENT.equals(c.getComponentName()) && HIVE_SERVICE.equals(c.getServiceName())); + .anyMatch(c -> MYSQL_SERVER_COMPONENT.equals(c.componentName()) && HIVE_SERVICE.equals(c.effectiveServiceName())); if (topologyContainsMysql && !hiveWantsMysql) { String errorMessage = String.format( diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java index caa3505de41..e41fe2708ff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java @@ -33,7 +33,7 @@ public class RejectUnknownComponents implements TopologyValidator { @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { String unknownComponents = topology.getComponents() - .map(ResolvedComponent::getComponentName) + .map(ResolvedComponent::componentName) .filter(c -> !RootComponent.AMBARI_SERVER.name().equals(c)) .filter(c -> !topology.getStack().getComponents().contains(c)) .collect(joining(", ")); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index 7fb80733bfe..b3a543032f9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -75,8 +75,8 @@ private Map>> validateRequiredPasswords(ClusterT groupEntry.getValue().getConfiguration().getFullProperties(3); Map> missingPropertiesInHostGroup = topology.getComponentsInHostGroup(hostGroupName) - .filter(component -> !RootComponent.AMBARI_SERVER.name().equals(component.getComponentName())) - .map(ResolvedComponent::getServiceType) + .filter(component -> !RootComponent.AMBARI_SERVER.name().equals(component.componentName())) + .map(ResolvedComponent::serviceType) .distinct() .flatMap(serviceType -> stack.getRequiredConfigurationProperties(serviceType, PropertyInfo.PropertyType.PASSWORD).stream()) .filter(property -> !propertyExists(groupProperties, property.getType(), property.getName())) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 9fbe39eb69c..84e3593b992 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.ALWAYS_APPLY; import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.NEVER_APPLY; @@ -8230,14 +8231,14 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration //todo: HG configs groupInfo.setConfiguration(hostGroup.configuration); - List componentList = new ArrayList<>(); - for (String componentName : hostGroup.components) { - componentList.add(new Component(componentName, null, stack.getServiceForComponent(componentName), null)); - } - Set components = componentList.stream() - .map(comp -> new ResolvedComponent(STACK_ID, comp.getServiceInstance(), comp.getServiceInstance(), comp)) + Set components = hostGroup.components.stream() + .map(name -> ResolvedComponent.builder(new Component(name)).stackId(STACK_ID).serviceType(stack.getServiceForComponent(name)).buildPartial()) .collect(toSet()); + List componentList = components.stream() + .map(ResolvedComponent::component) + .collect(toList()); + //create host group which is set on topology allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, componentList, EMPTY_CONFIG, "1")); hostGroupInfo.put(hostGroup.name, groupInfo); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 66893f38222..2458af46240 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -245,13 +245,15 @@ public void setUp() throws Exception { expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); + expect(topology.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); expect(topology.getServices()).andReturn(blueprintServices).anyTimes(); - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for both services + expect(topology.getComponents()).andAnswer(() -> Stream.of( + ResolvedComponent.builder(new Component("s1Component1")).stackId(STACK_ID).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("s1Component2")).stackId(STACK_ID).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("s2Component1")).stackId(STACK_ID).serviceType("service2").buildPartial() )).anyTimes(); - expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); - expect(blueprint.getSetting()).andReturn(setting).anyTimes(); + expect(topology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + expect(topology.getSetting()).andReturn(setting).anyTimes(); expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); @@ -338,7 +340,7 @@ public void testCreateAmbariResources() throws Exception { assertEquals(String.format("%s-%s", STACK_NAME, STACK_VERSION), clusterRequest.getStackVersion()); Set serviceGroupRequests = serviceGroupRequestCapture.getValue(); - Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), AmbariContext.DEFAULT_SERVICE_GROUP_NAME)); + Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), STACK_NAME)); assertEquals(expectedServiceGroupRequests, serviceGroupRequests); Collection serviceRequests = serviceRequestCapture.getValue(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index 9bab7c620cd..7d51906a356 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -31,11 +31,9 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -238,7 +236,7 @@ private Capture> testProcessWithKerberos(String blueprintP expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("testConfigType")).andReturn("KERBEROS").anyTimes(); @@ -247,22 +245,14 @@ private Capture> testProcessWithKerberos(String blueprintP expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); - Set services = new HashSet<>(); - services.add("HDFS"); - services.add("KERBEROS"); - services.add("ZOOKEPER"); + Set services = ImmutableSet.of("HDFS", "KERBEROS", "ZOOKEEPER"); expect(topology.getServices()).andReturn(services).anyTimes(); expect(stack.getConfiguration(services)).andReturn(stackDefaultConfig).once(); - List hdfsComponents = new ArrayList<>(); - hdfsComponents.add("NAMENODE"); - List kerberosComponents = new ArrayList<>(); - kerberosComponents.add("KERBEROS_CLIENT"); - List zookeeperComponents = new ArrayList<>(); - zookeeperComponents.add("ZOOKEEPER_SERVER"); - - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for all services + expect(topology.getComponents()).andAnswer(() -> Stream.of( + ResolvedComponent.builder(new Component("NAMENODE")).serviceType("HDFS").buildPartial(), + ResolvedComponent.builder(new Component("KERBEROS")).serviceType("KERBEROS_CLIENT").buildPartial(), + ResolvedComponent.builder(new Component("ZOOKEEPER_SERVER")).serviceType("ZOOKEEPER").buildPartial() )).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); @@ -331,38 +321,24 @@ public void testProcessClusterConfigRequestDontIncludeKerberosConfigs() throws E expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.singletonList("testConfigType")).anyTimes(); expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); - Set services = new HashSet<>(); - services.add("HDFS"); - services.add("KERBEROS"); - services.add("ZOOKEPER"); - expect(topology.getServices()).andReturn(services).anyTimes(); - - List hdfsComponents = new ArrayList<>(); - hdfsComponents.add("NAMENODE"); - List kerberosComponents = new ArrayList<>(); - kerberosComponents.add("KERBEROS_CLIENT"); - List zookeeperComponents = new ArrayList<>(); - zookeeperComponents.add("ZOOKEEPER_SERVER"); - + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "KERBEROS", "ZOOKEEPER")).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for all services - )).anyTimes(); + expect(topology.getComponents()).andAnswer(Stream::empty).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); - expect(topology.getClusterId()).andReturn(Long.valueOf(1)).anyTimes(); + expect(topology.getClusterId()).andReturn(1L).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); - expect(ambariContext.getClusterName(Long.valueOf(1))).andReturn("testCluster").anyTimes(); + expect(ambariContext.getClusterName(1L)).andReturn("testCluster").anyTimes(); expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections .emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 3e02d2c66a8..dab906242da 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -40,7 +40,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -82,6 +81,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -107,9 +107,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Mock(type = MockType.NICE) private Blueprint blueprint; - @Mock(type = MockType.NICE) - private ClusterTopology topology; - @Mock(type = MockType.NICE) private Stack stack; @@ -175,6 +172,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); @@ -258,10 +257,6 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for both services - )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -269,12 +264,10 @@ public void setup() throws Exception { expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(topology.getStack()).andReturn(stack).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -322,9 +315,21 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); @@ -332,7 +337,6 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index 772fd14d056..13d5b175a0c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -83,6 +82,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -108,9 +108,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Mock(type = MockType.NICE) private Blueprint blueprint; - @Mock(type = MockType.NICE) - private ClusterTopology topology; - @Mock(type = MockType.NICE) private Stack stack; @@ -172,6 +169,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -255,10 +255,6 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for both services - )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -266,12 +262,10 @@ public void setup() throws Exception { expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(topology.getStack()).andReturn(stack).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -319,9 +313,21 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(INSTALL_AND_START).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Arrays.asList("component1")) @@ -333,7 +339,6 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index 2f2cab89aac..d0841abdc17 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -41,7 +41,6 @@ import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.stream.Stream; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -83,6 +82,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -109,9 +109,6 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Mock(type = MockType.NICE) private Blueprint blueprint; - @Mock(type = MockType.NICE) - private ClusterTopology topology; - @Mock(type = MockType.NICE) private Stack stack; @@ -174,6 +171,9 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -258,10 +258,6 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for both services - )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -269,16 +265,18 @@ public void setup() throws Exception { expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(topology.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(topology.getStack()).andReturn(stack).anyTimes(); expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(topology.isValidConfigType(anyString())).andReturn(true).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() - expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); - expect(stack.getAllConfigurationTypes("service2")).andReturn(Arrays.asList("service2-site", "service2-env")).anyTimes(); + List configTypes1 = Arrays.asList("service1-site", "service1-env"); + List configTypes2 = Arrays.asList("service2-site", "service2-env"); + expect(stack.getConfigurationTypes("service1")).andReturn(configTypes1).anyTimes(); + expect(stack.getConfigurationTypes("service2")).andReturn(configTypes2).anyTimes(); + expect(stack.getAllConfigurationTypes("service1")).andReturn(configTypes1).anyTimes(); + expect(stack.getAllConfigurationTypes("service2")).andReturn(configTypes2).anyTimes(); expect(stack.getAutoDeployInfo("component1")).andReturn(null).anyTimes(); expect(stack.getAutoDeployInfo("component2")).andReturn(null).anyTimes(); expect(stack.getAutoDeployInfo("component3")).andReturn(null).anyTimes(); @@ -322,9 +320,21 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(INSTALL_ONLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); @@ -333,7 +343,6 @@ public void setup() throws Exception { expect(group1.getName()).andReturn("group1").anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(topology.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java index 6df1559af98..b0c3958d322 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java @@ -116,8 +116,9 @@ public void setup() throws Exception { expect(clusterTopology.getClusterId()).andReturn(clusterId).anyTimes(); expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_ONLY).anyTimes(); expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getBlueprintName()).andReturn("blueprintDef").anyTimes(); + expect(clusterTopology.getSetting()).andReturn(setting).anyTimes(); expect(blueprint.getName()).andReturn("blueprintDef").anyTimes(); - expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.shouldSkipFailure()).andReturn(true).anyTimes(); PowerMock.reset(AmbariServer.class); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java similarity index 92% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java index 7ab4e4762e8..6235aae4666 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ComponentResolverTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java @@ -33,7 +33,7 @@ import com.google.common.collect.ImmutableSet; -public class ComponentResolverTest { +public class StackComponentResolverTest { private final TopologyValidator validator = new RejectUnknownComponents(); private final ClusterTopology topology = createNiceMock(ClusterTopology.class); @@ -73,7 +73,9 @@ public void rejectsUnknownComponents() throws Exception { } private void componentsInTopologyAre(String... components) { - expect(topology.getComponents()).andReturn(Stream.empty()).anyTimes(); // FIXME + expect(topology.getComponents()).andReturn(Stream.of(components) + .map(name -> ResolvedComponent.builder(new Component(name)).buildPartial()) + ).anyTimes(); replay(topology); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 49e2023550e..9f3a1f04825 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -92,6 +92,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; /** @@ -178,6 +179,9 @@ public class TopologyManagerTest { @Mock private TopologyValidatorService topologyValidatorService; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -263,7 +267,10 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(clusterTopologyMock.getComponents()).andReturn(Stream.of( - // FIXME add ResolvedComponents for both services + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component3")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); @@ -274,9 +281,9 @@ public void setup() throws Exception { expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(clusterTopologyMock.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); expect(clusterTopologyMock.getStack()).andReturn(stack).anyTimes(); - expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -313,7 +320,10 @@ public void setup() throws Exception { expect(request.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); expect(request.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of()).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); expect(clusterTopologyMock.containsMasterComponent("group1")).andReturn(true).anyTimes(); @@ -337,6 +347,7 @@ public void setup() throws Exception { expect(logicalRequest.getReservedHosts()).andReturn(Collections.singleton("host1")).anyTimes(); expect(logicalRequest.getRequestStatus()).andReturn(requestStatusResponse).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), isNull(), anyLong()); @@ -389,12 +400,12 @@ public void setup() throws Exception { @After public void tearDown() { PowerMock.verify(System.class); - verify(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + verify(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); PowerMock.reset(System.class); - reset(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + reset(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); } @@ -550,7 +561,7 @@ private void requestFinished() { } private void replayAll() { - replay(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + replay(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, configurationRequest, configurationRequest2, configurationRequest3, executor, persistedState, clusterTopologyMock, securityConfigurationFactory, credentialStoreService, clusterController, resourceProvider, mockFuture, requestStatusResponse, logicalRequest, settingDAO, diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java index 5d796bdcd73..9e6ac130962 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java @@ -136,7 +136,7 @@ public void setup() { dependenciesConditionInfos1.add(dependencyConditionInfo1); dependenciesConditionInfos1.add(dependencyConditionInfo2); - expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes(); + expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); } @After diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index 12ca8e3f065..805edaddf5b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -18,11 +18,12 @@ import static org.easymock.EasyMock.replay; import java.util.Arrays; -import java.util.stream.Stream; import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -133,7 +134,9 @@ private void topologyHasMysql(boolean hasMysql) { if (hasMysql) { components.add("MYSQL_SERVER"); } - expect(topology.getComponents()).andReturn(Stream.empty()).anyTimes(); // FIXME + expect(topology.getComponents()).andReturn(components.build().stream() + .map(name -> ResolvedComponent.builder(new Component(name)).serviceType("HIVE").buildPartial()) + ).anyTimes(); expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); replay(topology); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java index 1f28e936680..f7ce215ccd8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java @@ -26,15 +26,18 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.stream.Stream; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -78,8 +81,6 @@ public class RequiredPasswordValidatorTest extends EasyMockSupport { private static final Map hostGroups = new HashMap<>(); private static final Map hostGroupInfo = new HashMap<>(); - private static final Collection group1Components = new HashSet<>(); - private static final Collection group2Components = new HashSet<>(); private static final Collection service1Components = new HashSet<>(); private static final Collection service2Components = new HashSet<>(); private static final Collection service3Components = new HashSet<>(); @@ -123,13 +124,6 @@ public void setup() { hostGroups.put("group1", group1); hostGroups.put("group2", group2); - group1Components.add("component1"); - group1Components.add("component2"); - group1Components.add("component3"); - - group2Components.add("component1"); - group2Components.add("component4"); - service1Components.add("component1"); service1Components.add("component2"); service2Components.add("component3"); @@ -151,13 +145,15 @@ public void setup() { expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); expect(topology.getStack()).andReturn(stack).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); - - expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); - expect(stack.getServiceForComponent("component2")).andReturn("service1").anyTimes(); - expect(stack.getServiceForComponent("component3")).andReturn("service2").anyTimes(); - expect(stack.getServiceForComponent("component4")).andReturn("service3").anyTimes(); + expect(topology.getComponentsInHostGroup("group1")).andReturn(Stream.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial() + )).anyTimes(); + expect(topology.getComponentsInHostGroup("group2")).andReturn(Stream.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service3").buildPartial() + )).anyTimes(); expect(stack.getRequiredConfigurationProperties("service1", PropertyInfo.PropertyType.PASSWORD)).andReturn(service1RequiredPwdConfigs).anyTimes(); expect(stack.getRequiredConfigurationProperties("service2", PropertyInfo.PropertyType.PASSWORD)).andReturn(service2RequiredPwdConfigs).anyTimes(); From a179dc172da23a84b4ced2216726e1dc626a2aea Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 20 Feb 2018 15:47:29 +0100 Subject: [PATCH 19/25] Use ResolvedComponent for host component creation --- .../ambari/server/topology/AmbariContext.java | 30 +++++---------- .../server/topology/ResolvedComponent.java | 4 +- .../tasks/PersistHostResourcesTask.java | 2 +- .../server/topology/AmbariContextTest.java | 38 +++---------------- 4 files changed, 17 insertions(+), 57 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 4b0ace6a4d7..21d303f32b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -26,7 +26,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -398,7 +397,7 @@ private void createAmbariServiceAndComponentResources(ClusterTopology topology, } } - public void createAmbariHostResources(long clusterId, String hostName, Collection components) { + public void createAmbariHostResources(long clusterId, String hostName, Stream components) { Host host; try { host = getController().getClusters().getHost(hostName); @@ -408,7 +407,7 @@ public void createAmbariHostResources(long clusterId, String hostName, Collecti "Unable to obtain host instance '%s' when persisting host resources", hostName)); } - Cluster cluster = null; + final Cluster cluster; try { cluster = getController().getClusters().getClusterById(clusterId); } catch (AmbariException e) { @@ -430,26 +429,15 @@ public void createAmbariHostResources(long clusterId, String hostName, Collecti hostName, e.toString()), e); } - final Set requests = new HashSet<>(); + final Set requests = components + .filter(component -> !component.componentName().equals(RootComponent.AMBARI_SERVER.name())) + .map(component -> new ServiceComponentHostRequest(clusterName, component.effectiveServiceGroupName(), component.effectiveServiceName(), component.componentName(), hostName, null)) + .collect(toSet()); - for (Component component : components) { - String componentName = component.getName(); - String serviceName = component.getServiceInstance(); - try { - if (cluster.getService(serviceName) != null && !componentName.equals(RootComponent.AMBARI_SERVER.name())) { - requests.add(new ServiceComponentHostRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, serviceName, componentName, hostName, null)); - } - } catch (AmbariException e) { - LOG.warn("Service already deleted from cluster: {}", serviceName); - } - } try { - RetryHelper.executeWithRetry(new Callable() { - @Override - public Object call() throws Exception { - getController().createHostComponents(requests); - return null; - } + RetryHelper.executeWithRetry(() -> { + getController().createHostComponents(requests); + return null; }); } catch (AmbariException e) { LOG.error("Unable to create host component resource for host {}", hostName, e); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java index c4d62a10393..85e524bcefe 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -45,14 +45,14 @@ public interface ResolvedComponent { * @return service group name if it set, otherwise defaults to the stack name */ default String effectiveServiceGroupName() { - return serviceGroupName().orElse(stackId().getStackName()); + return serviceGroupName().orElseGet(() -> stackId().getStackName()); } /** * @return service name if it set, otherwise defaults to the service type (eg. ZOOKEEPER) */ default String effectiveServiceName() { - return serviceName().orElse(serviceType()); + return serviceName().orElseGet(this::serviceType); } /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java index 58f174c3209..4ae136b6dba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java @@ -44,7 +44,7 @@ public void runTask() { String hostName = hostRequest.getHostName(); LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostName); long clusterId = hostRequest.getClusterId(); - clusterTopology.getAmbariContext().createAmbariHostResources(clusterId, hostName, hostRequest.getHostGroup().getComponents()); + clusterTopology.getAmbariContext().createAmbariHostResources(clusterId, hostName, clusterTopology.getComponentsInHostGroup(hostRequest.getHostgroupName())); LOG.info("HostRequest: Exiting RESOURCE_CREATION task for host: {}", hostName); } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 2458af46240..a03bd83c937 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -35,7 +35,6 @@ import static org.junit.Assert.fail; import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -402,8 +401,6 @@ public void testCreateAmbariHostResources() throws Exception { hostResourceProvider.createHosts(anyObject(Request.class)); expectLastCall().once(); - expect(cluster.getService("service1")).andReturn(mockService1).times(2); - expect(cluster.getService("service2")).andReturn(mockService1).once(); Capture> requestsCapture = EasyMock.newCapture(); expect(controller.createHostComponents(capture(requestsCapture))).andReturn(null).once(); @@ -411,41 +408,16 @@ public void testCreateAmbariHostResources() throws Exception { replayAll(); // test - Collection components = new ArrayList<>(); - components.add(new Component("component1", "mpack", "service1", null)); - components.add(new Component("component2", "mpack", "service1", null)); - components.add(new Component("component3", "mpack", "service2", null)); - + Stream components = Stream.of( + ResolvedComponent.builder(new Component("component1", "mpack", "service1", null)).buildPartial(), + ResolvedComponent.builder(new Component("component2", "mpack", "service1", null)).buildPartial(), + ResolvedComponent.builder(new Component("component3", "mpack", "service2", null)).buildPartial() + ); context.createAmbariHostResources(CLUSTER_ID, "host1", components); assertEquals(requestsCapture.getValue().size(), 3); } - @Test - public void testCreateAmbariHostResourcesWithMissingService() throws Exception { - // expectations - expect(cluster.getServices()).andReturn(clusterServices).anyTimes(); - - hostResourceProvider.createHosts(anyObject(Request.class)); - expectLastCall().once(); - expect(cluster.getService("service1")).andReturn(mockService1).times(2); - Capture> requestsCapture = EasyMock.newCapture(); - - expect(controller.createHostComponents(capture(requestsCapture))).andReturn(null).once(); - - replayAll(); - - // test - Collection components = new ArrayList<>(); - components.add(new Component("component1", "mpack", "service1", null)); - components.add(new Component("component2", "mpack", "service1", null)); - components.add(new Component("component3", "mpack", "service2", null)); - - context.createAmbariHostResources(CLUSTER_ID, "host1", components); - - assertEquals(requestsCapture.getValue().size(), 2); - } - @Test public void testRegisterHostWithConfigGroup_createNewConfigGroup() throws Exception { // test specific expectations From 53d64d512815685143c855150bbfc730e224b881 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 20 Feb 2018 18:50:06 +0100 Subject: [PATCH 20/25] Workaround for missing default cluster settings --- .../ambari/server/topology/AmbariContext.java | 14 ++++++++++++++ .../ambari/server/topology/AmbariContextTest.java | 11 ++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 21d303f32b0..aedba04d38c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -337,6 +337,8 @@ private void createAmbariClusterResource(String clusterName, Set stackI getController().createCluster(clusterRequest); return null; }); + + addDefaultClusterSettings(clusterName); } catch (AmbariException e) { LOG.error("Failed to create Cluster resource: ", e); if (e.getCause() instanceof DuplicateResourceException) { @@ -347,6 +349,18 @@ private void createAmbariClusterResource(String clusterName, Set stackI } } + // FIXME temporarily add default cluster settings -- should be provided by ClusterImpl itself + private void addDefaultClusterSettings(String clusterName) throws AmbariException { + Cluster cluster = getController().getClusters().getCluster(clusterName); + Set> properties = getController().getAmbariMetaInfo().getClusterProperties().stream() + .map(p -> Pair.of(p.getName(), p.getValue())) + .collect(toSet()); + + for (Pair p : properties) { + cluster.addClusterSetting(p.getKey(), p.getValue()); + } + } + private void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { Set serviceGroupRequests = topology.getComponents() .map(c -> new ServiceGroupRequest(clusterName, c.effectiveServiceGroupName())) diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index a03bd83c937..0167ff0a2c5 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.capture; @@ -44,6 +45,7 @@ import java.util.Set; import java.util.stream.Stream; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigGroupRequest; @@ -107,6 +109,7 @@ public class AmbariContextTest { private static final AmbariContext context = new AmbariContext(); private static final AmbariManagementController controller = createNiceMock(AmbariManagementController.class); + private static final AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class); private static final ClusterController clusterController = createStrictMock(ClusterController.class); private static final HostResourceProvider hostResourceProvider = createStrictMock(HostResourceProvider.class); private static final ServiceGroupResourceProvider serviceGroupResourceProvider = createStrictMock(ServiceGroupResourceProvider.class); @@ -140,7 +143,7 @@ public class AmbariContextTest { @Before public void setUp() throws Exception { - reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, + reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); @@ -264,6 +267,8 @@ public void setUp() throws Exception { expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(controller.getConfigHelper()).andReturn(configHelper).anyTimes(); + expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); + expect(metaInfo.getClusterProperties()).andReturn(emptySet()).anyTimes(); expect(clusters.getCluster(CLUSTER_NAME)).andReturn(cluster).anyTimes(); expect(clusters.getClusterById(CLUSTER_ID)).andReturn(cluster).anyTimes(); @@ -289,13 +294,13 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, + verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } private void replayAll() { - replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, + replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } From 1516b064a87d0c4e1df93064ca25c759fa6a1f94 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 22 Feb 2018 16:14:24 +0100 Subject: [PATCH 21/25] Avoid storing null-null as StackId --- .../server/topology/BlueprintFactory.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index aee21e1edd5..77d366bf3e4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -41,6 +41,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.Function; @@ -112,8 +113,12 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu Collection mpackInstances = createMpackInstances(properties); if (mpackInstances.isEmpty()) { - StackId stackId = getStackId(properties); - mpackInstances = Collections.singleton(new MpackInstance(stackId.getStackName(), stackId.getStackVersion(), null, null, Configuration.createEmpty())); + Optional stackId = getStackId(properties); + if (stackId.isPresent()) { + String stackName = stackId.get().getStackName(); + String stackVersion = stackId.get().getStackVersion(); + mpackInstances = Collections.singleton(new MpackInstance(stackName, stackVersion, null, null, Configuration.createEmpty())); + } } Set stackIds = mpackInstances.stream() .map(MpackInstance::getStackId) @@ -142,10 +147,12 @@ public static Collection createMpackInstances(Map } } - private static StackId getStackId(Map properties) throws NoSuchStackException { - String stackName = String.valueOf(properties.get(STACK_NAME_PROPERTY_ID)); - String stackVersion = String.valueOf(properties.get(STACK_VERSION_PROPERTY_ID)); - return new StackId(stackName, stackVersion); + private static Optional getStackId(Map properties) throws NoSuchStackException { + Object stackName = properties.get(STACK_NAME_PROPERTY_ID); + Object stackVersion = properties.get(STACK_VERSION_PROPERTY_ID); + return stackName != null && stackVersion != null + ? Optional.of(new StackId(stackName.toString(), stackVersion.toString())) + : Optional.empty(); } public StackDefinition composeStacks(Set stackIds) { From 07dcdea899e2908fc0f742fdb738dd97113aced6 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 22 Feb 2018 16:33:30 +0100 Subject: [PATCH 22/25] Remove unused code --- .../server/topology/BlueprintFactory.java | 28 +------------------ .../server/topology/BlueprintFactoryTest.java | 4 +-- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 77d366bf3e4..0ac76df322a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -50,10 +50,7 @@ import javax.inject.Inject; import javax.inject.Provider; -import org.apache.ambari.server.ObjectNotFoundException; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.Stack; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.stack.NoSuchStackException; @@ -76,12 +73,10 @@ public class BlueprintFactory { private final Provider blueprintDAO; private final ConfigurationFactory configFactory = new ConfigurationFactory(); - private final StackFactory stackFactory; @Inject - public BlueprintFactory(Provider blueprintDAO, StackFactory stackFactory) { + public BlueprintFactory(Provider blueprintDAO) { this.blueprintDAO = blueprintDAO; - this.stackFactory = stackFactory; } public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException { @@ -155,19 +150,6 @@ private static Optional getStackId(Map properties) thro : Optional.empty(); } - public StackDefinition composeStacks(Set stackIds) { - Set stacks = stackIds.stream() - .map(this::createStack) - .collect(toSet()); - StackDefinition composite = StackDefinition.of(stacks); - - // temporary check - verifyStackDefinitionsAreDisjoint(composite.getServices().stream(), "Service", composite::getStacksForService); - verifyStackDefinitionsAreDisjoint(composite.getComponents().stream(), "Component", composite::getStacksForComponent); - - return composite; - } - /** * Verify that each item in items is defined by only one stack. * @@ -191,14 +173,6 @@ static void verifyStackDefinitionsAreDisjoint(Stream items, String type, } } - private Stack createStack(StackId stackId) { - try { - return stackFactory.createStack(stackId); - } catch (ObjectNotFoundException e) { - throw new NoSuchStackException(stackId); - } - } - //todo: Move logic to HostGroupImpl @SuppressWarnings("unchecked") private Collection processHostGroups(Map properties) { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index 03791879b5c..ff163d37671 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -63,7 +63,7 @@ public class BlueprintFactoryTest { private BlueprintDAO dao = createStrictMock(BlueprintDAO.class); private BlueprintEntity entity = createStrictMock(BlueprintEntity.class); private BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); - private BlueprintFactory testFactory = new BlueprintFactory(Providers.of(dao), __ -> stack); + private BlueprintFactory testFactory = new BlueprintFactory(Providers.of(dao)); @Before public void init() throws Exception { @@ -129,7 +129,7 @@ public void testGetBlueprint_NotFound() throws Exception { expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null).once(); replay(dao, entity, configEntity); - BlueprintFactory factory = new BlueprintFactory(Providers.of(dao), null); + BlueprintFactory factory = new BlueprintFactory(Providers.of(dao)); assertNull(factory.getBlueprint(BLUEPRINT_NAME)); } From 9859d74f719b237ef4d9bfc798adb9f416f8cb15 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 23 Feb 2018 15:20:44 +0100 Subject: [PATCH 23/25] Address review comments (simplify, document) --- .../ambari/server/controller/internal/Stack.java | 1 + .../ambari/server/topology/AmbariContext.java | 9 +++------ .../ambari/server/topology/ComponentResolver.java | 10 ++++++++++ .../server/topology/StackComponentResolver.java | 4 ---- .../topology/validators/GplPropertiesValidator.java | 13 ++----------- 5 files changed, 16 insertions(+), 21 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index af34c711985..d08e5726485 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -597,6 +597,7 @@ private void parseExcludedConfigurations(ServiceInfo stackServiceResponse) { excludedConfigurationTypes.put(stackServiceResponse.getName(), stackServiceResponse.getExcludedConfigTypes()); } + //todo: This information should be specified in the stack definition. private void registerConditionalDependencies() { dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index c2901ad689e..f53b767fd7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -92,6 +92,7 @@ import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; @@ -352,12 +353,8 @@ private void createAmbariClusterResource(String clusterName, Set stackI // FIXME temporarily add default cluster settings -- should be provided by ClusterImpl itself private void addDefaultClusterSettings(String clusterName) throws AmbariException { Cluster cluster = getController().getClusters().getCluster(clusterName); - Set> properties = getController().getAmbariMetaInfo().getClusterProperties().stream() - .map(p -> Pair.of(p.getName(), p.getValue())) - .collect(toSet()); - - for (Pair p : properties) { - cluster.addClusterSetting(p.getKey(), p.getValue()); + for (PropertyInfo p : getController().getAmbariMetaInfo().getClusterProperties()) { + cluster.addClusterSetting(p.getName(), p.getValue()); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java index ac74768b35c..1738d88b424 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java @@ -3,8 +3,18 @@ import java.util.Map; import java.util.Set; +/** + * Resolves all incompletely specified host group components in the topology: + * finds stack and/or service type that each component is defined in. + */ public interface ComponentResolver { + /** + * @return the set resolved components for each host group (the map's keys are host group names) + * @throws IllegalArgumentException if the components cannot be unambiguously resolved + * (eg. if some component is not known, or if there are multiple component with the same name and + * the request does not specify which one to select) + */ Map> resolveComponents(BlueprintBasedClusterProvisionRequest request); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java index 661ebc8db72..9d524d027c0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java @@ -36,10 +36,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Strings; -/** - * Resolves all incompletely specified host group components in the topology: - * finds stack and/or service type that each component is defined in. - */ public class StackComponentResolver implements ComponentResolver { private static final Logger LOG = LoggerFactory.getLogger(StackComponentResolver.class); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java index 467908e648c..5ce4940fe50 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java @@ -74,17 +74,8 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { Map> clusterConfigurations = topology.getConfiguration().getProperties(); if (clusterConfigurations != null) { - for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { - String configType = configEntry.getKey(); - if (!CORE_SITE.equals(configType)) { - continue; - } - - Map properties = configEntry.getValue(); - if (properties == null) { - continue; - } - + Map properties = clusterConfigurations.get(CORE_SITE); + if (properties != null) { for (String propertyName : PROPERTY_NAMES) { String propertyValue = properties.get(propertyName); if (propertyValue != null && propertyValue.contains(LZO_CODEC_CLASS)) { From 6fcdbc1b7d65f46b0f9fc4426703ca5562ae57ac Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 28 Feb 2018 12:53:14 +0100 Subject: [PATCH 24/25] Fix unit tests --- .../BlueprintConfigurationProcessor.java | 2 +- .../server/topology/ClusterTopologyImpl.java | 4 +- .../server/agent/AgentResourceTest.java | 9 ++ .../render/ClusterBlueprintRendererTest.java | 98 +++++++------------ .../StackAdvisorBlueprintProcessorTest.java | 12 +-- .../ambari/server/utils/StageUtilsTest.java | 9 ++ 6 files changed, 65 insertions(+), 69 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index e3efb36568c..92e146de507 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -562,7 +562,7 @@ private void doFilterPriorToExport(Configuration configuration) { String clusterName = clusterTopology.getAmbariContext().getClusterName(clusterTopology.getClusterId()); Cluster cluster = clusterTopology.getAmbariContext().getController().getClusters().getCluster(clusterName); authToLocalPerClusterMap = new HashMap<>(); - authToLocalPerClusterMap.put(Long.valueOf(clusterTopology.getClusterId()), clusterTopology.getAmbariContext().getController().getKerberosHelper().getKerberosDescriptor(cluster, false).getAllAuthToLocalProperties()); + authToLocalPerClusterMap.put(clusterTopology.getClusterId(), clusterTopology.getAmbariContext().getController().getKerberosHelper().getKerberosDescriptor(cluster, false).getAllAuthToLocalProperties()); } catch (AmbariException e) { LOG.error("Error while getting authToLocal properties. ", e); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index 09d614d3b9f..95814579f19 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -79,8 +79,8 @@ public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topology provisionRequest = null; defaultPassword = null; - stackIds = ImmutableSet.of(); - stack = null; + stackIds = topologyRequest.getBlueprint().getStackIds(); + stack = ambariContext.composeStacks(stackIds); resolvedComponents = ImmutableMap.of(); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java index 3d9a013d561..6bc2c93bf4d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java @@ -80,8 +80,14 @@ import org.apache.ambari.server.state.scheduler.RequestExecutionImpl; import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl; +import org.apache.ambari.server.topology.ComponentResolver; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; +import org.apache.ambari.server.topology.StackComponentResolver; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.easymock.EasyMock; @@ -335,6 +341,9 @@ protected void configure() { bind(PersistedState.class).toInstance(createNiceMock(PersistedState.class)); bind(RoleCommandOrderProvider.class).to(CachedRoleCommandOrderProvider.class); bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); + bind(ComponentResolver.class).to(StackComponentResolver.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); + bind(StackFactory.class).to(DefaultStackFactory.class); } private void installDependencies() { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java index 1deb685c73a..0cb0e88b373 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java @@ -19,10 +19,8 @@ package org.apache.ambari.server.api.query.render; import static java.util.stream.Collectors.toList; -import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -63,10 +61,14 @@ import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; +import org.apache.ambari.server.controller.spi.NoSuchResourceException; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.DesiredConfig; @@ -102,6 +104,7 @@ public class ClusterBlueprintRendererTest { private static final ClusterTopology topology = createNiceMock(ClusterTopology.class); + private static final ClusterTopology topologyWithKerberos = createNiceMock(ClusterTopology.class); private static final ClusterController clusterController = createNiceMock(ClusterControllerImpl.class); private static final AmbariContext ambariContext = createNiceMock(AmbariContext.class); @@ -124,9 +127,14 @@ public class ClusterBlueprintRendererTest { private static final Configuration clusterConfig = new Configuration(clusterProps, clusterAttributes); public static final StackId STACK_ID = new StackId("HDP", "1.3.3"); + private static final ResourceProvider artifactResourceProvider = createNiceMock(ResourceProvider.class); + private static final Resource artifactResource = createNiceMock(Resource.class); @Before public void setup() throws Exception { + PowerMock.mockStatic(AmbariContext.class); + expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + expect(AmbariContext.getController()).andReturn(controller).anyTimes(); Map clusterTypeProps = new HashMap<>(); clusterProps.put("test-type-one", clusterTypeProps); @@ -163,10 +171,10 @@ public void setup() throws Exception { groupInfoMap.put("host_group_1", group1Info); groupInfoMap.put("host_group_2", group2Info); - expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); - expect(topology.getStack()).andReturn(stack).anyTimes(); + setupTopology(topology, groupInfoMap); + expect(topology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); + setupTopology(topologyWithKerberos, groupInfoMap); + expect(topologyWithKerberos.isClusterKerberosEnabled()).andReturn(true).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); expect(blueprint.getHostGroup("host_group_1")).andReturn(group1).anyTimes(); @@ -181,11 +189,6 @@ public void setup() throws Exception { expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); - expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); - expect(topology.getClusterId()).andReturn(1L).anyTimes(); - PowerMock.mockStatic(AmbariServer.class); - expect(AmbariServer.getController()).andReturn(controller).anyTimes(); - PowerMock.replay(AmbariServer.class); expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes(); expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); @@ -194,63 +197,40 @@ public void setup() throws Exception { properties.add("core-site/hadoop.security.auth_to_local"); expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes(); expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes(); - replay(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); - } - - private void setupMocksForKerberosEnabledCluster() throws Exception { - - AmbariContext ambariContext = createNiceMock(AmbariContext.class); - expect(ambariContext.getClusterName(anyLong())).andReturn("clusterName").anyTimes(); - - PowerMock.mockStatic(AmbariContext.class); - expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); - expect(AmbariContext.getController()).andReturn(controller).anyTimes(); - - reset(topology); - HostGroupInfo group1Info = new HostGroupInfo("host_group_1"); - group1Info.addHost("host1"); - group1Info.setConfiguration(emptyConfiguration); - HostGroupInfo group2Info = new HostGroupInfo("host_group_2"); - Map groupInfoMap = new HashMap<>(); - group2Info.addHosts(Arrays.asList("host2", "host3")); - group2Info.setConfiguration(emptyConfiguration); - groupInfoMap.put("host_group_1", group1Info); - groupInfoMap.put("host_group_2", group2Info); + setupKerberosDescriptorArtifact(); - expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); - expect(topology.getClusterId()).andReturn(new Long(1)).anyTimes(); - expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); - expect(topology.isClusterKerberosEnabled()).andReturn(true).anyTimes(); - - ResourceProvider resourceProvider = createStrictMock(ResourceProvider.class); - expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(resourceProvider).once(); - - Resource resource = createStrictMock(Resource.class); - Set result = Collections.singleton(resource); + PowerMock.replay(AmbariContext.class); + replay(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + } - expect(resourceProvider.getResources((Request) anyObject(Request.class), - (Predicate) anyObject(Predicate.class))).andReturn(result).once(); + @After + public void tearDown() { + verify(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + reset(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + PowerMock.reset(AmbariContext.class); + } + private void setupKerberosDescriptorArtifact() throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Map> resourcePropertiesMap = new HashMap<>(); resourcePropertiesMap.put(ArtifactResourceProvider.ARTIFACT_DATA_PROPERTY, Collections.emptyMap()); Map propertiesMap = new HashMap<>(); propertiesMap.put("testProperty", "testValue"); resourcePropertiesMap.put(ArtifactResourceProvider.ARTIFACT_DATA_PROPERTY + "/properties", propertiesMap); - expect(resource.getPropertiesMap()).andReturn(resourcePropertiesMap).once(); - - PowerMock.replay(AmbariContext.class); - replay(ambariContext, topology, clusterController, resource, resourceProvider); + expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes(); + expect(artifactResourceProvider.getResources(anyObject(Request.class), anyObject(Predicate.class))).andReturn(Collections.singleton(artifactResource)).anyTimes(); + expect(artifactResource.getPropertiesMap()).andReturn(resourcePropertiesMap).anyTimes(); } - - @After - public void tearDown() { - verify(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); - reset(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); + private void setupTopology(ClusterTopology topology, Map groupInfoMap) { + expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(topology.getClusterId()).andReturn(1L).anyTimes(); } @Test @@ -441,13 +421,10 @@ public void testFinalizeProperties__instance_noComponentNode() { @Test public void testFinalizeResult_kerberos() throws Exception{ - - setupMocksForKerberosEnabledCluster(); - Result result = new ResultImpl(true); createClusterResultTree(result.getResultTree()); - ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology); + ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topologyWithKerberos); Result blueprintResult = renderer.finalizeResult(result); TreeNode blueprintTree = blueprintResult.getResultTree(); @@ -463,6 +440,7 @@ public void testFinalizeResult_kerberos() throws Exception{ assertEquals(STACK_ID.getStackVersion(), properties.get("Blueprints").get("stack_version")); Map securityProperties = (Map) properties.get("Blueprints").get("security"); + assertNotNull(securityProperties); assertEquals("KERBEROS", securityProperties.get("type")); assertNotNull(((Map) securityProperties.get("kerberos_descriptor")).get("properties")); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java index 4fa28acafca..2abc1522799 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java @@ -88,7 +88,7 @@ public void testAdviseConfiguration() throws StackAdvisorException, Configuratio expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY).anyTimes(); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); @@ -127,7 +127,7 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApply() throws StackAdvi expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); @@ -167,7 +167,7 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApplyWhenNoUserInputForD expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); @@ -205,7 +205,7 @@ public void testAdviseConfigurationWith_ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES).anyTimes(); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); @@ -241,7 +241,7 @@ public void testAdviseConfigurationWhenConfigurationRecommendFails() throws Stac expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); @@ -274,7 +274,7 @@ public void testAdviseConfigurationWhenConfigurationRecommendHasInvalidResponse( expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index d43a712bf92..18637d4505e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -82,9 +82,15 @@ import org.apache.ambari.server.state.cluster.ClusterFactory; import org.apache.ambari.server.state.host.HostFactory; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.topology.ComponentResolver; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; +import org.apache.ambari.server.topology.StackComponentResolver; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; import org.easymock.EasyMockSupport; @@ -135,6 +141,9 @@ protected void configure() { bind(MpackManagerFactory.class).toInstance(createNiceMock(MpackManagerFactory.class)); bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); bind(RootLevelSettingsManagerFactory.class).toInstance(createNiceMock(RootLevelSettingsManagerFactory.class)); + bind(ComponentResolver.class).to(StackComponentResolver.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); + bind(StackFactory.class).to(DefaultStackFactory.class); install(new FactoryModuleBuilder().build(ExecutionCommandWrapperFactory.class)); install(new FactoryModuleBuilder().implement(Config.class, ConfigImpl.class).build(ConfigFactory.class)); install(new FactoryModuleBuilder().build(ConfigureClusterTaskFactory.class)); From 9be4a6a2fccbc1174b0f443d34e93c3d616042a7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 28 Feb 2018 13:07:54 +0100 Subject: [PATCH 25/25] Log level back to debug --- .../controller/internal/BlueprintConfigurationProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 92e146de507..5ea81c34418 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2907,11 +2907,11 @@ private Collection setupHDFSProxyUsers(Configuration configuration, Set< private void addExcludedConfigProperties(Configuration configuration, Set configTypesUpdated, StackDefinition stack) { Collection blueprintServices = clusterTopology.getServices(); - LOG.info("Handling excluded properties for blueprint services: {}", blueprintServices); + LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices); for (String blueprintService : blueprintServices) { - LOG.info("Handling excluded properties for blueprint service: {}", blueprintService); + LOG.debug("Handling excluded properties for blueprint service: {}", blueprintService); Set excludedConfigTypes = stack.getExcludedConfigurationTypes(blueprintService); if (excludedConfigTypes.isEmpty()) {