diff --git a/ambari-project/pom.xml b/ambari-project/pom.xml index 7bb01b5360c..ff338e85a35 100644 --- a/ambari-project/pom.xml +++ b/ambari-project/pom.xml @@ -485,6 +485,13 @@ swagger-maven-plugin ${swagger.maven.plugin.version} + + org.inferred + freebuilder + 1.14.8 + true + provided + diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 7ceae4872c3..31d97ed050a 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -1644,6 +1644,10 @@ 0.1.10 test + + org.inferred + freebuilder + diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java index deb13db607a..e518de7b68f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java @@ -197,7 +197,7 @@ private Resource createBlueprintResource(TreeNode clusterNode) { BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); - Set stackIds = topology.getBlueprint().getStackIds(); + Set stackIds = topology.getStackIds(); if (stackIds.size() == 1) { StackId stackId = Iterables.getOnlyElement(stackIds); blueprintResource.setProperty("Blueprints/stack_name", stackId.getStackName()); @@ -210,8 +210,7 @@ private Resource createBlueprintResource(TreeNode clusterNode) { try { String clusterName = topology.getAmbariContext().getClusterName(topology.getClusterId()); - Map kerberosDescriptor = getKerberosDescriptor(topology.getAmbariContext() - .getClusterController(), clusterName); + Map kerberosDescriptor = getKerberosDescriptor(AmbariContext.getClusterController(), clusterName); if (kerberosDescriptor != null) { securityConfigMap.put(SecurityConfigurationFactory.KERBEROS_DESCRIPTOR_PROPERTY_ID, kerberosDescriptor); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 410903540cb..baf80f37d35 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@ -719,13 +719,20 @@ public Collection getStacks(String stackName) throws AmbariException * @param stackId * the stack id * @return the stack info - * @throws AmbariException */ - public StackInfo getStack(StackId stackId) throws AmbariException { + public StackInfo getStack(StackId stackId) throws StackAccessException { return getStack(stackId.getStackName(), stackId.getStackVersion()); } - public StackInfo getStack(String stackName, String version) throws AmbariException { + public boolean isKnownStack(StackId stackId) { + try { + return getStack(stackId) != null; + } catch (StackAccessException e) { + return false; + } + } + + public StackInfo getStack(String stackName, String version) throws StackAccessException { StackInfo stackInfoResult = stackManager.getStack(stackName, version); if (stackInfoResult == null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java index 94ba563ed1f..87b64a56e81 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java @@ -31,7 +31,6 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.AdvisedConfiguration; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.HostGroup; @@ -76,7 +75,7 @@ public static void init(StackAdvisorHelper instance) { * @param userProvidedConfigurations User configurations of cluster provided in Blueprint + Cluster template */ public void adviseConfiguration(ClusterTopology clusterTopology, Map> userProvidedConfigurations) throws ConfigurationTopologyException { - for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) { + for (StackId stackId : clusterTopology.getStackIds()) { StackAdvisorRequest request = createStackAdvisorRequest(clusterTopology, stackId, StackAdvisorRequestType.CONFIGURATIONS); try { RecommendationResponse response = stackAdvisorHelper.recommend(request); @@ -96,7 +95,7 @@ private StackAdvisorRequest createStackAdvisorRequest(ClusterTopology clusterTop hgHostsMap); return StackAdvisorRequest.StackAdvisorRequestBuilder .forStack(stackId) - .forServices(clusterTopology.getBlueprint().getStack().getServices(stackId)) + .forServices(clusterTopology.getStack().getServices(stackId)) .forHosts(gatherHosts(clusterTopology)) .forHostsGroupBindings(gatherHostGroupBindings(clusterTopology)) .forHostComponents(gatherHostGroupComponents(clusterTopology)) @@ -176,12 +175,11 @@ private void addAdvisedConfigurationsToTopology(RecommendationResponse response, Map recommendedConfigurations = response.getRecommendations().getBlueprint().getConfigurations(); - Blueprint blueprint = topology.getBlueprint(); for (Map.Entry configEntry : recommendedConfigurations.entrySet()) { String configType = configEntry.getKey(); // add recommended config type only if related service is present in Blueprint - if (blueprint.isValidConfigType(configType)) { + if (topology.isValidConfigType(configType)) { BlueprintConfigurations blueprintConfig = filterBlueprintConfig(configType, configEntry.getValue(), userProvidedConfigurations, topology); topology.getAdvisedConfigurations().put(configType, new AdvisedConfiguration( diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index e93277e541f..fbe80ed0226 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -81,7 +81,6 @@ import org.apache.ambari.server.metrics.system.MetricsService; import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.PersistenceType; -import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.dao.ClusterDAO; import org.apache.ambari.server.orm.dao.GroupDAO; import org.apache.ambari.server.orm.dao.MetainfoDAO; @@ -948,7 +947,6 @@ public void performStaticInjection() { injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory .class), injector.getInstance(Gson.class)); HostResourceProvider.setTopologyManager(injector.getInstance(TopologyManager.class)); - BlueprintFactory.init(injector.getInstance(BlueprintDAO.class)); BaseClusterRequest.init(injector.getInstance(BlueprintFactory.class)); AmbariContext.init(injector.getInstance(HostRoleCommandFactory.class)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index 357f1b61d86..0ae2676ee5a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -169,12 +169,16 @@ import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.BlueprintValidatorImpl; +import org.apache.ambari.server.topology.ComponentResolver; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; import org.apache.ambari.server.topology.PersistedStateImpl; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.StackComponentResolver; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.PasswordUtils; import org.apache.ambari.server.view.ViewInstanceHandlerList; import org.eclipse.jetty.server.SessionIdManager; @@ -414,6 +418,7 @@ protected void configure() { bind(SecurityConfigurationFactory.class).in(Scopes.SINGLETON); bind(PersistedState.class).to(PersistedStateImpl.class); + bind(ComponentResolver.class).to(StackComponentResolver.class); // factory to create LoggingRequestHelper instances for LogSearch integration bind(LoggingRequestHelperFactory.class).to(LoggingRequestHelperFactoryImpl.class); @@ -541,7 +546,8 @@ private void installFactories() { bind(RegistryFactory.class).to(RegistryFactoryImpl.class); bind(HostRoleCommandFactory.class).to(HostRoleCommandFactoryImpl.class); bind(SecurityHelper.class).toInstance(SecurityHelperImpl.getInstance()); - bind(BlueprintValidator.class).to(BlueprintValidatorImpl.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); + bind(StackFactory.class).to(DefaultStackFactory.class); bind(BlueprintFactory.class); install(new FactoryModuleBuilder().implement(AmbariEvent.class, Names.named("userCreated"), UserCreatedEvent.class).build(AmbariEventFactory.class)); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java index f59eb984e7f..0c73933c753 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceComponentRequest.java @@ -19,6 +19,8 @@ package org.apache.ambari.server.controller; +import java.util.Objects; + public class ServiceComponentRequest { private String clusterName; // REF @@ -145,4 +147,29 @@ public String toString() { "desiredState=%s, recoveryEnabled=%s, componentCategory=%s]", clusterName, serviceGroupName, serviceName, clusterName, desiredState, recoveryEnabled, componentCategory); } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ServiceComponentRequest other = (ServiceComponentRequest) obj; + + return Objects.equals(clusterName, other.clusterName) && + Objects.equals(serviceGroupName, other.serviceGroupName) && + Objects.equals(serviceName, other.serviceName) && + Objects.equals(componentCategory, other.componentCategory) && + Objects.equals(componentName, other.componentName) && + Objects.equals(desiredState, other.desiredState) && + Objects.equals(recoveryEnabled, other.recoveryEnabled); + } + + @Override + public int hashCode() { + return Objects.hash(clusterName, serviceGroupName, serviceName, componentCategory, componentName, desiredState, recoveryEnabled); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java index 70e5240c38a..4516b555069 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceRequest.java @@ -17,6 +17,8 @@ */ package org.apache.ambari.server.controller; +import java.util.Objects; + import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.StackId; @@ -196,4 +198,32 @@ public String toString() { public void setResolvedRepository(RepositoryVersionEntity repositoryVersion) { resolvedRepository = repositoryVersion; } public RepositoryVersionEntity getResolvedRepository() { return resolvedRepository; } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + ServiceRequest other = (ServiceRequest) obj; + + return Objects.equals(clusterName, other.clusterName) && + Objects.equals(serviceGroupName, other.serviceGroupName) && + Objects.equals(serviceType, other.serviceType) && + Objects.equals(serviceName, other.serviceName) && + Objects.equals(desiredState, other.desiredState) && + Objects.equals(maintenanceState, other.maintenanceState) && + Objects.equals(credentialStoreEnabled, other.credentialStoreEnabled) && + Objects.equals(credentialStoreSupported, other.credentialStoreSupported) && + Objects.equals(desiredStackId, other.desiredStackId) && + Objects.equals(desiredRepositoryVersionId, other.desiredRepositoryVersionId); + } + + @Override + public int hashCode() { + return Objects.hash(clusterName, serviceGroupName, serviceType, serviceName, desiredState, maintenanceState, credentialStoreEnabled, credentialStoreSupported, desiredStackId, desiredRepositoryVersionId); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 8da1f51ee2c..5ea81c34418 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -44,12 +44,10 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.AdvisedConfiguration; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.Configuration; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.validators.UnitValidatedProperty; import org.apache.commons.lang.StringUtils; @@ -71,6 +69,11 @@ public class BlueprintConfigurationProcessor { private static final Logger LOG = LoggerFactory.getLogger(BlueprintConfigurationProcessor.class); + /** + * Compiled regex for "%HOSTGROUP::...%" token. + */ + public static final Pattern HOST_GROUP_PLACEHOLDER_PATTERN = Pattern.compile("%HOSTGROUP::(\\S+?)%"); + private final static String COMMAND_RETRY_ENABLED_PROPERTY_NAME = "command_retry_enabled"; private final static String COMMANDS_TO_RETRY_PROPERTY_NAME = "commands_to_retry"; @@ -165,6 +168,23 @@ public class BlueprintConfigurationProcessor { private static Set configPropertiesWithHASupport = new HashSet<>(Arrays.asList("fs.defaultFS", "hbase.rootdir", "instance.volumes", "policymgr_external_url", "xasecure.audit.destination.hdfs.dir")); + public static boolean isNameNodeHAEnabled(Map> configurationProperties) { + return configurationProperties.containsKey("hdfs-site") && + (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") || + configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices")); + } + + /** + * Static convenience function to determine if Yarn ResourceManager HA is enabled + * @param configProperties configuration properties for this cluster + * @return true if Yarn ResourceManager HA is enabled + * false if Yarn ResourceManager HA is not enabled + */ + public static boolean isYarnResourceManagerHAEnabled(Map> configProperties) { + return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled") + && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true"); + } + /** * Statically-defined list of filters to apply on property exports. * This will initially be used to filter out the Ranger Passwords, but @@ -279,7 +299,7 @@ private boolean containsHostFromHostGroups(String configType, String propertyNam return false; } // check fir bp import - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(propertyValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(propertyValue); if (m.find()) { return true; } @@ -350,7 +370,8 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept // set of properties (copy) doesn't include the removed properties. If an updater // removes a property other than the property it is registered for then we will // have an issue as it won't be removed from the clusterProps map as it is a copy. - Map> clusterProps = clusterConfig.getFullProperties(); + Map> configProperties = clusterConfig.getFullProperties(); + Map> clusterProps = configProperties; for (Map> updaterMap : createCollectionOfUpdaters()) { for (Map.Entry> entry : updaterMap.entrySet()) { String type = entry.getKey(); @@ -398,10 +419,10 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept } //todo: lots of hard coded HA rules included here - if (clusterTopology.isNameNodeHAEnabled()) { + if (isNameNodeHAEnabled(configProperties)) { // add "dfs.internal.nameservices" if it's not specified - Map hdfsSiteConfig = clusterConfig.getFullProperties().get("hdfs-site"); + Map hdfsSiteConfig = configProperties.get("hdfs-site"); String nameservices = hdfsSiteConfig.get("dfs.nameservices"); String int_nameservices = hdfsSiteConfig.get("dfs.internal.nameservices"); if(int_nameservices == null && nameservices != null) { @@ -430,7 +451,7 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept setStackToolsAndFeatures(clusterConfig, configTypesUpdated); setRetryConfiguration(clusterConfig, configTypesUpdated); setupHDFSProxyUsers(clusterConfig, configTypesUpdated); - addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getBlueprint().getStack()); + addExcludedConfigProperties(clusterConfig, configTypesUpdated, clusterTopology.getStack()); trimProperties(clusterConfig, clusterTopology); @@ -438,8 +459,7 @@ public Set doUpdateForClusterCreate() throws ConfigurationTopologyExcept } private void trimProperties(Configuration clusterConfig, ClusterTopology clusterTopology) { - Blueprint blueprint = clusterTopology.getBlueprint(); - StackDefinition stack = blueprint.getStack(); + StackDefinition stack = clusterTopology.getStack(); Map> configTypes = clusterConfig.getFullProperties(); for (String configType : configTypes.keySet()) { @@ -487,15 +507,16 @@ private static boolean shouldPropertyBeStoredWithDefault(String propertyName) { */ public void doUpdateForBlueprintExport() { // HA configs are only processed in cluster configuration, not HG configurations - if (clusterTopology.isNameNodeHAEnabled()) { + Map> configProperties = clusterTopology.getConfiguration().getFullProperties(); + if (isNameNodeHAEnabled(configProperties)) { doNameNodeHAUpdate(); } - if (clusterTopology.isYarnResourceManagerHAEnabled()) { + if (isYarnResourceManagerHAEnabled(configProperties)) { doYarnResourceManagerHAUpdate(); } - if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + if (isOozieServerHAEnabled(configProperties)) { doOozieServerHAUpdate(); } @@ -541,7 +562,7 @@ private void doFilterPriorToExport(Configuration configuration) { String clusterName = clusterTopology.getAmbariContext().getClusterName(clusterTopology.getClusterId()); Cluster cluster = clusterTopology.getAmbariContext().getController().getClusters().getCluster(clusterName); authToLocalPerClusterMap = new HashMap<>(); - authToLocalPerClusterMap.put(Long.valueOf(clusterTopology.getClusterId()), clusterTopology.getAmbariContext().getController().getKerberosHelper().getKerberosDescriptor(cluster, false).getAllAuthToLocalProperties()); + authToLocalPerClusterMap.put(clusterTopology.getClusterId(), clusterTopology.getAmbariContext().getController().getKerberosHelper().getKerberosDescriptor(cluster, false).getAllAuthToLocalProperties()); } catch (AmbariException e) { LOG.error("Error while getting authToLocal properties. ", e); } @@ -549,7 +570,7 @@ private void doFilterPriorToExport(Configuration configuration) { for (Map.Entry> configEntry : properties.entrySet()) { String type = configEntry.getKey(); try { - clusterTopology.getBlueprint().getStack().getServiceForConfigType(type); + clusterTopology.getStack().getServiceForConfigType(type); } catch (IllegalArgumentException illegalArgumentException) { LOG.error(new StringBuilder(String.format("Error encountered while trying to obtain the service name for config type [%s]. ", type)) .append("Further processing on this config type will be skipped. ") @@ -623,8 +644,7 @@ private void doRecommendConfigurations(Configuration configuration, Set * @param advisedConfigurations advised configuration instance */ private void doFilterStackDefaults(Map advisedConfigurations) { - Blueprint blueprint = clusterTopology.getBlueprint(); - Configuration stackDefaults = blueprint.getStack().getConfiguration(blueprint.getServices()); + Configuration stackDefaults = clusterTopology.getStack().getConfiguration(clusterTopology.getServices()); Map> stackDefaultProps = stackDefaults.getProperties(); for (Map.Entry adConfEntry : advisedConfigurations.entrySet()) { AdvisedConfiguration advisedConfiguration = adConfEntry.getValue(); @@ -702,15 +722,16 @@ private void doRemovePropertiesIfNeeded(Configuration configuration, private Collection>> createCollectionOfUpdaters() { Collection>> updaters = allUpdaters; - if (clusterTopology.isNameNodeHAEnabled()) { + Map> configProperties = clusterTopology.getConfiguration().getFullProperties(); + if (isNameNodeHAEnabled(configProperties)) { updaters = addNameNodeHAUpdaters(updaters); } - if (clusterTopology.isYarnResourceManagerHAEnabled()) { + if (isYarnResourceManagerHAEnabled(configProperties)) { updaters = addYarnResourceManagerHAUpdaters(updaters); } - if (isOozieServerHAEnabled(clusterTopology.getConfiguration().getFullProperties())) { + if (isOozieServerHAEnabled(configProperties)) { updaters = addOozieServerHAUpdaters(updaters); } @@ -1381,7 +1402,7 @@ public String updateForClusterCreate(String propertyName, ClusterTopology topology) { //todo: getHostStrings - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(origValue); if (m.find()) { String hostGroupName = m.group(1); @@ -1406,7 +1427,7 @@ public Collection getRequiredHostGroups(String propertyName, Map> properties, ClusterTopology topology) { //todo: getHostStrings - Matcher m = HostGroup.HOSTGROUP_REGEX.matcher(origValue); + Matcher m = HOST_GROUP_PLACEHOLDER_PATTERN.matcher(origValue); if (m.find()) { String hostGroupName = m.group(1); return Collections.singleton(hostGroupName); @@ -1461,7 +1482,7 @@ public String updateForClusterCreate(String propertyName, topology.getHostAssignmentsForComponent(component).iterator().next(), properties); } else { //todo: extract all hard coded HA logic - Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component); + Cardinality cardinality = topology.getStack().getCardinality(component); // if no matching host groups are found for a component whose configuration // is handled by this updater, check the stack first to determine if // zero is a valid cardinality for this component. This is necessary @@ -1470,7 +1491,8 @@ public String updateForClusterCreate(String propertyName, if (matchingGroupCount == 0 && cardinality.isValidCount(0)) { return origValue; } else { - if (topology.isNameNodeHAEnabled() && isComponentNameNode() && (matchingGroupCount == 2)) { + Map> configProperties = topology.getConfiguration().getFullProperties(); + if (isComponentNameNode() && (matchingGroupCount == 2) && isNameNodeHAEnabled(configProperties)) { // if this is the defaultFS property, it should reflect the nameservice name, // rather than a hostname (used in non-HA scenarios) if (properties.containsKey("core-site") && properties.get("core-site").get("fs.defaultFS").equals(origValue)) { @@ -1496,13 +1518,13 @@ public String updateForClusterCreate(String propertyName, } - if (topology.isNameNodeHAEnabled() && isComponentSecondaryNameNode() && (matchingGroupCount == 0)) { + if (isComponentSecondaryNameNode() && (matchingGroupCount == 0) && isNameNodeHAEnabled(configProperties)) { // if HDFS HA is enabled, then no replacement is necessary for properties that refer to the SECONDARY_NAMENODE // eventually this type of information should be encoded in the stacks return origValue; } - if (topology.isYarnResourceManagerHAEnabled() && isComponentResourceManager() && (matchingGroupCount == 2)) { + if (isComponentResourceManager() && (matchingGroupCount == 2) && isYarnResourceManagerHAEnabled(configProperties)) { if (!origValue.contains("localhost")) { // if this Yarn property is a FQDN, then simply return it return origValue; @@ -1570,7 +1592,7 @@ public Collection getRequiredHostGroups(String propertyName, if (matchingGroupCount != 0) { return new HashSet<>(matchingGroups); } else { - Cardinality cardinality = topology.getBlueprint().getStack().getCardinality(component); + Cardinality cardinality = topology.getStack().getCardinality(component); // if no matching host groups are found for a component whose configuration // is handled by this updater, return an empty set if (! cardinality.isValidCount(0)) { @@ -2064,7 +2086,7 @@ public Collection getRequiredHostGroups(String propertyName, while (m.find()) { String groupName = m.group(1); - if (!topology.getBlueprint().getHostGroups().containsKey(groupName)) { + if (!topology.getHostGroupInfo().containsKey(groupName)) { throw new IllegalArgumentException( "Unable to match blueprint host group token to a host group: " + groupName); } @@ -2582,7 +2604,7 @@ public String updateForClusterCreate(String propertyName, } } - boolean isAtlasInCluster = topology.getBlueprint().getServices().contains("ATLAS"); + boolean isAtlasInCluster = topology.getServices().contains("ATLAS"); boolean isAtlasHiveHookEnabled = Boolean.parseBoolean(properties.get("hive-env").get("hive.atlas.hook")); // Append atlas hook if not already present. @@ -2613,7 +2635,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("ATLAS")) { + if (topology.getServices().contains("ATLAS")) { // if original value is not set or is the default "primary" set the cluster id if (origValue == null || origValue.trim().isEmpty() || origValue.equals("primary")) { //use cluster id because cluster name may change @@ -2648,7 +2670,7 @@ public String updateForClusterCreate(String propertyName, String origValue, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("ATLAS")) { + if (topology.getServices().contains("ATLAS")) { String host = topology.getHostAssignmentsForComponent("ATLAS_SERVER").iterator().next(); boolean tlsEnabled = Boolean.parseBoolean(properties.get("application-properties").get("atlas.enableTLS")); @@ -2706,7 +2728,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) { + if (topology.getServices().contains("AMBARI_METRICS")) { final String amsReporterClass = "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReporterClass; @@ -2737,7 +2759,7 @@ public String updateForClusterCreate(String propertyName, Map> properties, ClusterTopology topology) { - if (topology.getBlueprint().getServices().contains("AMBARI_METRICS")) { + if (topology.getServices().contains("AMBARI_METRICS")) { final String amsReportesClass = "org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter"; if (origValue == null || origValue.isEmpty()) { return amsReportesClass; @@ -2829,7 +2851,7 @@ private Collection setupHDFSProxyUsers(Configuration configuration, Set< // AMBARI-5206 final Map userProps = new HashMap<>(); - Collection services = clusterTopology.getBlueprint().getServices(); + Collection services = clusterTopology.getServices(); if (services.contains("HDFS")) { // only add user properties to the map for // services actually included in the blueprint definition @@ -2883,7 +2905,7 @@ private Collection setupHDFSProxyUsers(Configuration configuration, Set< * @param stack */ private void addExcludedConfigProperties(Configuration configuration, Set configTypesUpdated, StackDefinition stack) { - Collection blueprintServices = clusterTopology.getBlueprint().getServices(); + Collection blueprintServices = clusterTopology.getServices(); LOG.debug("Handling excluded properties for blueprint services: {}", blueprintServices); @@ -2987,7 +3009,7 @@ private void setStackToolsAndFeatures(Configuration configuration, Set c ); try { - for (StackId stackId : clusterTopology.getBlueprint().getStackIds()) { + for (StackId stackId : clusterTopology.getStackIds()) { Map> defaultStackProperties = configHelper.getDefaultStackProperties(stackId); if (defaultStackProperties.containsKey(CLUSTER_ENV_CONFIG_TYPE_NAME)) { Map clusterEnvDefaultProperties = defaultStackProperties.get(CLUSTER_ENV_CONFIG_TYPE_NAME); @@ -3106,7 +3128,7 @@ private static class StackPropertyTypeFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); final String serviceName = stack.getServiceForConfigType(configType); return !(stack.isPasswordProperty(serviceName, configType, propertyName) || stack.isKerberosPrincipalNameProperty(serviceName, configType, propertyName)); @@ -3203,7 +3225,7 @@ private static abstract class DependencyFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); Configuration configuration = topology.getConfiguration(); final String serviceName = stack.getServiceForConfigType(configType); @@ -3317,7 +3339,7 @@ private static class HDFSNameNodeHAFilter implements PropertyFilter { * namenode. */ private final Set setOfHDFSPropertyNamesNonHA = - Collections.unmodifiableSet(new HashSet<>(Arrays.asList("dfs.namenode.http-address", "dfs.namenode.https-address", "dfs.namenode.rpc-address"))); + ImmutableSet.of("dfs.namenode.http-address", "dfs.namenode.https-address", "dfs.namenode.rpc-address"); /** @@ -3332,13 +3354,8 @@ private static class HDFSNameNodeHAFilter implements PropertyFilter { */ @Override public boolean isPropertyIncluded(String propertyName, String propertyValue, String configType, ClusterTopology topology) { - if (topology.isNameNodeHAEnabled()) { - if (setOfHDFSPropertyNamesNonHA.contains(propertyName)) { - return false; - } - } - - return true; + return !setOfHDFSPropertyNamesNonHA.contains(propertyName) + || !isNameNodeHAEnabled(topology.getConfiguration().getFullProperties()); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 760d90f89c0..8b45c4110d6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -59,12 +59,10 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; -import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.JsonUtils; import org.apache.ambari.server.utils.SecretReference; import org.slf4j.Logger; @@ -545,19 +543,8 @@ public Void invoke() throws AmbariException { blueprint.getName()); } - try { - validator.validateRequiredProperties(blueprint); - } catch (InvalidTopologyException | GPLLicenseNotAcceptedException e) { - throw new IllegalArgumentException("Blueprint configuration validation failed: " + e.getMessage(), e); - } - - String validateTopology = requestInfoProps.get(VALIDATE_TOPOLOGY_PROPERTY_ID); - if (validateTopology == null || ! validateTopology.equalsIgnoreCase("false")) { - try { - validator.validateTopology(blueprint); - } catch (InvalidTopologyException e) { - throw new IllegalArgumentException(e.getMessage()); - } + if (shouldValidate(requestInfoProps)) { + validator.validate(blueprint); } LOG.info("Creating Blueprint, name=" + blueprint.getName()); @@ -574,6 +561,11 @@ public Void invoke() throws AmbariException { }; } + private static boolean shouldValidate(Map requestInfoProps) { + String validateTopology = requestInfoProps.get("validate_topology"); + return validateTopology == null || Boolean.parseBoolean(validateTopology); + } + /** * The structure of blueprints is evolving where multiple resource * structures are to be supported. This class abstracts the population diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java index 7a1f517d4e8..f03231bfac1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java @@ -538,12 +538,6 @@ private RequestStatusResponse processBlueprintCreate(Map propert throw new IllegalArgumentException("Invalid Cluster Creation Template: " + e, e); } - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.NONE && - createClusterRequest.getBlueprint().getSecurity() != null && createClusterRequest.getBlueprint().getSecurity() - .getType() == SecurityType.KERBEROS) { - throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); - } - try { return topologyManager.provisionCluster(createClusterRequest); } catch (InvalidTopologyException e) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java index 75baf300bcf..333a20aebf8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/CompositeStack.java @@ -27,6 +27,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ConfigHelper; @@ -207,10 +209,10 @@ public String getServiceForComponent(String component) { } @Override - public Collection getServicesForComponents(Collection components) { + @Nonnull + public Stream> getServicesForComponent(String component) { return stacks.stream() - .flatMap(m -> m.getServicesForComponents(components).stream()) - .collect(toSet()); + .flatMap(stack -> stack.getServicesForComponent(component)); } @Override @@ -246,15 +248,6 @@ public Collection getDependenciesForComponent(String component) .collect(toSet()); } - @Override - public String getConditionalServiceForDependency(DependencyInfo dependency) { - return stacks.stream() - .map(m -> m.getConditionalServiceForDependency(dependency)) - .filter(Objects::nonNull) - .findAny() - .orElse(null); - } - @Override public String getExternalComponentConfig(String component) { return stacks.stream() diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java index 13ae732cec5..151d9a7c3ee 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; -import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.api.util.TreeNode; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; @@ -83,7 +83,7 @@ public ExportBlueprintRequest(TreeNode clusterNode) throws InvalidTopo Collection exportedHostGroups = processHostGroups(clusterNode.getChild("hosts")); createHostGroupInfo(exportedHostGroups); - createBlueprint(exportedHostGroups, parseStack(clusterResource)); + createBlueprint(exportedHostGroups, createStack(parseStack(clusterResource))); } public String getClusterName() { @@ -135,11 +135,11 @@ private void createBlueprint(Collection exportedHostGroups, S componentList.add(new Component(component)); } - hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList, + hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), componentList, exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality()))); } ImmutableSet stackIds = ImmutableSet.of(stack.getStackId()); - blueprint = new BlueprintImpl(bpName, hostGroups, stack, stackIds, Collections.emptySet(), configuration, null, null); + blueprint = new BlueprintImpl(bpName, hostGroups, stackIds, Collections.emptySet(), configuration, null, null); } private void createHostGroupInfo(Collection exportedHostGroups) { @@ -153,15 +153,15 @@ private void createHostGroupInfo(Collection exportedHostGroup } - private Stack parseStack(Resource clusterResource) throws InvalidTopologyTemplateException { - String[] stackTokens = String.valueOf(clusterResource.getPropertyValue( - ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID)).split("-"); + private StackId parseStack(Resource clusterResource) { + return new StackId(String.valueOf(clusterResource.getPropertyValue(ClusterResourceProvider.CLUSTER_VERSION_PROPERTY_ID))); + } + private Stack createStack(StackId stackId) throws InvalidTopologyTemplateException { try { - return new Stack(stackTokens[0], stackTokens[1], controller); - } catch (AmbariException e) { - throw new InvalidTopologyTemplateException(String.format( - "The specified stack doesn't exist: name=%s version=%s", stackTokens[0], stackTokens[1])); + return new Stack(stackId, controller.getAmbariMetaInfo()); + } catch (StackAccessException e) { + throw new InvalidTopologyTemplateException(String.format("The specified stack doesn't exist: %s", stackId)); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index 115973c9ec5..01ba9157c07 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -18,8 +18,9 @@ package org.apache.ambari.server.controller.internal; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import static org.apache.ambari.server.topology.TopologyManager.KDC_ADMIN_CREDENTIAL; -import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -29,8 +30,11 @@ import org.apache.ambari.server.api.predicate.InvalidQueryException; import org.apache.ambari.server.security.encryption.CredentialStoreType; import org.apache.ambari.server.stack.NoSuchStackException; +import org.apache.ambari.server.state.SecurityType; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileBuilder; import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfileEvaluationException; +import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.ConfigurationFactory; @@ -40,13 +44,11 @@ import org.apache.ambari.server.topology.ManagementPackMapping; import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.NoSuchBlueprintException; +import org.apache.ambari.server.topology.ProvisionRequest; import org.apache.ambari.server.topology.SecurityConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Enums; import com.google.common.base.Optional; import com.google.common.base.Strings; @@ -55,7 +57,7 @@ * Request for provisioning a cluster. */ @SuppressWarnings("unchecked") -public class ProvisionClusterRequest extends BaseClusterRequest { +public class ProvisionClusterRequest extends BaseClusterRequest implements ProvisionRequest { /** * host groups property name */ @@ -129,8 +131,6 @@ public class ProvisionClusterRequest extends BaseClusterRequest { public static final String MANAGEMENT_PACK_MAPPINGS_PROPERTY = "management_pack_mappings"; - public static final String MPACK_INSTANCES_PROPERTY = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID; - public static final String MPACK_INSTANCE_PROPERTY = "mpack_instance"; public static final String COMPONENT_NAME_PROPERTY = "component_name"; @@ -163,7 +163,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest { private final String quickLinksProfileJson; - private Collection mpackInstances; + private final Collection mpackInstances; + private final Set stackIds; private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class); @@ -199,21 +200,23 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura } this.securityConfiguration = securityConfiguration; + this.credentialsMap = parseCredentials(properties); + if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS && getCredentialsMap().get(KDC_ADMIN_CREDENTIAL) == null) { + throw new InvalidTopologyTemplateException(KDC_ADMIN_CREDENTIAL + " is missing from request."); + } - Configuration configuration = configurationFactory.getConfiguration( - (Collection>) properties.get(CONFIGURATIONS_PROPERTY)); + Configuration configuration = configurationFactory.getConfiguration((Collection>) properties.get(CONFIGURATIONS_PROPERTY)); configuration.setParentConfiguration(blueprint.getConfiguration()); setConfiguration(configuration); parseHostGroupInfo(properties); - this.credentialsMap = parseCredentials(properties); - this.configRecommendationStrategy = parseConfigRecommendationStrategy(properties); setProvisionAction(parseProvisionAction(properties)); - processMpackInstances(properties); + mpackInstances = BlueprintFactory.createMpackInstances(properties); + stackIds = mpackInstances.stream().map(MpackInstance::getStackId).collect(toSet()); // FIXME persist these try { this.quickLinksProfileJson = processQuickLinksProfile(properties); @@ -222,21 +225,6 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura } } - private void processMpackInstances(Map properties) throws InvalidTopologyTemplateException { - if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) { - ObjectMapper mapper = new ObjectMapper(); - mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); - try { - String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY)); - this.mpackInstances = mapper.readValue(mpackInstancesJson, - new TypeReference>() {}); - } - catch (IOException ex) { - throw new InvalidTopologyTemplateException("Cannot process mpack instances.", ex); - } - } - } - private String processQuickLinksProfile(Map properties) throws QuickLinksProfileEvaluationException { Object globalFilters = properties.get(QUICKLINKS_PROFILE_FILTERS_PROPERTY); Object serviceFilters = properties.get(QUICKLINKS_PROFILE_SERVICES_PROPERTY); @@ -291,6 +279,7 @@ public void setClusterName(String clusterName) { this.clusterName = clusterName; } + @Override public ConfigRecommendationStrategy getConfigRecommendationStrategy() { return configRecommendationStrategy; } @@ -530,11 +519,18 @@ public String getQuickLinksProfileJson() { return quickLinksProfileJson; } + @Override public String getDefaultPassword() { return defaultPassword; } - public Collection getMpackInstances() { + @Override + public Set getStackIds() { + return stackIds; + } + + @Override + public Collection getMpacks() { return mpackInstances; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index 77422eb2fa4..d08e5726485 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -26,8 +26,10 @@ import java.util.Set; import java.util.stream.Stream; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.AmbariManagementController; +import javax.annotation.Nonnull; + +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.ReadOnlyConfigurationResponse; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; @@ -41,6 +43,7 @@ import org.apache.ambari.server.state.ValueAttributesInfo; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; +import org.apache.commons.lang3.tuple.Pair; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -74,12 +77,6 @@ public class Stack implements StackDefinition { private Map> dependencies = new HashMap<>(); - /** - * Map of dependency to conditional service - */ - private Map dependencyConditionalServiceMap = - new HashMap<>(); - /** * Map of database component name to configuration property which indicates whether * the database in to be managed or if it is an external non-managed instance. @@ -127,8 +124,8 @@ public class Stack implements StackDefinition { private Map> excludedConfigurationTypes = new HashMap<>(); - public Stack(String name, String version, AmbariManagementController ctrl) throws AmbariException { // FIXME remove or at least change to use metainfo directly - this(ctrl.getAmbariMetaInfo().getStack(name, version)); + public Stack(StackId stackId, AmbariMetaInfo metaInfo) throws StackAccessException { + this(metaInfo.getStack(stackId)); } public Stack(StackInfo stackInfo) { @@ -170,10 +167,6 @@ public StackId getStackId() { return new StackId(getName(), getVersion()); } - Map getDependencyConditionalServiceMap() { - return dependencyConditionalServiceMap; - } - @Override public Set getStackIds() { return ImmutableSet.of(getStackId()); @@ -380,13 +373,12 @@ public String getServiceForComponent(String component) { } @Override - public Collection getServicesForComponents(Collection components) { - Set services = new HashSet<>(); - for (String component : components) { - services.add(getServiceForComponent(component)); - } - - return services; + @Nonnull + public Stream> getServicesForComponent(String component) { + String service = getServiceForComponent(component); + return service != null + ? Stream.of(Pair.of(getStackId(), service)) + : Stream.empty(); } @Override @@ -422,11 +414,6 @@ public Collection getDependenciesForComponent(String component) Collections.emptySet(); } - @Override - public String getConditionalServiceForDependency(DependencyInfo dependency) { - return dependencyConditionalServiceMap.get(dependency); - } - @Override public String getExternalComponentConfig(String component) { return dbDependencyInfo.get(component); @@ -610,11 +597,8 @@ private void parseExcludedConfigurations(ServiceInfo stackServiceResponse) { excludedConfigurationTypes.put(stackServiceResponse.getName(), stackServiceResponse.getExcludedConfigTypes()); } - /** - * Register conditional dependencies. - */ //todo: This information should be specified in the stack definition. - void registerConditionalDependencies() { + private void registerConditionalDependencies() { dbDependencyInfo.put("MYSQL_SERVER", "global/hive_database"); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java index 82aaa693ee2..3f9c74f933a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackDefinition.java @@ -23,6 +23,8 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.Nonnull; + import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.DependencyInfo; @@ -30,6 +32,8 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.validators.DependencyAndCardinalityValidator; +import org.apache.commons.lang3.tuple.Pair; /** * Encapsulates stack information. @@ -200,13 +204,10 @@ public interface StackDefinition { String getServiceForComponent(String component); /** - * Get the names of the services which contains the specified components. - * - * @param components collection of components - * - * @return collection of services which contain the specified components + * Get (stackID, service) pairs which contain the specified component in this stack. */ - Collection getServicesForComponents(Collection components); + @Nonnull + Stream> getServicesForComponent(String component); /** * Obtain the service name which corresponds to the specified configuration. @@ -232,16 +233,6 @@ public interface StackDefinition { //todo: full dependency graph Collection getDependenciesForComponent(String component); - /** - * Get the service, if any, that a component dependency is conditional on. - * - * @param dependency dependency to get conditional service for - * - * @return conditional service for provided component or null if dependency - * is not conditional on a service - */ - String getConditionalServiceForDependency(DependencyInfo dependency); - /** * Get the custom "descriptor" that is used to decide whether component * is a managed or non-managed dependency. The descriptor is formatted as: @@ -250,7 +241,7 @@ public interface StackDefinition { * * @param component component to get dependency information for * @return the descriptor of form "config_type/property_name" - * @see org.apache.ambari.server.topology.BlueprintValidatorImpl#isDependencyManaged + * @see DependencyAndCardinalityValidator#isDependencyManaged */ String getExternalComponentConfig(String component); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java index bb02b44febc..0e57599fd87 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java @@ -49,7 +49,7 @@ public String updateForClusterCreate(String propertyName, String origValue, Map> properties, ClusterTopology topology) { - PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName); + PropertyUnit stackUnit = PropertyUnit.of(topology.getStack(), serviceName, configType, propertyName); PropertyValue value = PropertyValue.of(propertyName, origValue); if (value.hasUnit(stackUnit)) { return value.toString(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 6fbb59e810a..f53b767fd7e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -19,13 +19,13 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toSet; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -34,6 +34,8 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; +import java.util.function.Function; +import java.util.stream.Stream; import javax.annotation.Nullable; import javax.inject.Inject; @@ -43,6 +45,7 @@ import org.apache.ambari.server.DuplicateResourceException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; +import org.apache.ambari.server.StackAccessException; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.actionmanager.HostRoleStatus; @@ -68,6 +71,7 @@ import org.apache.ambari.server.controller.internal.ServiceGroupDependencyResourceProvider; import org.apache.ambari.server.controller.internal.ServiceGroupResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; +import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.controller.internal.VersionDefinitionResourceProvider; import org.apache.ambari.server.controller.predicate.EqualsPredicate; @@ -80,6 +84,7 @@ import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.security.authorization.AuthorizationException; +import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; @@ -87,15 +92,18 @@ import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.DesiredConfig; import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.utils.RetryHelper; +import org.apache.commons.lang3.tuple.Pair; import org.apache.directory.api.util.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; @@ -121,6 +129,9 @@ public enum TaskType {INSTALL, START} @Inject ConfigFactory configFactory; + @Inject + StackFactory stackFactory; + @Inject RepositoryVersionDAO repositoryVersionDAO; @@ -215,7 +226,7 @@ public void createAmbariResources(ClusterTopology topology, String clusterName, String repoVersionString, Long repoVersionId) { Map repoVersionByStack = new HashMap<>(); - Set stackIds = topology.getBlueprint().getStackIds(); + Set stackIds = topology.getStackIds(); for (StackId stackId : stackIds) { RepositoryVersionEntity repoVersion = null; if (stackIds.size() == 1) { @@ -234,8 +245,7 @@ public void createAmbariResources(ClusterTopology topology, String clusterName, } } - StackId stackId = Iterables.getFirst(topology.getBlueprint().getStackIds(), null); - createAmbariClusterResource(clusterName, stackId, securityType); + createAmbariClusterResource(clusterName, topology.getStackIds(), securityType); createAmbariServiceAndComponentResources(topology, clusterName, repoVersionByStack); } @@ -319,19 +329,17 @@ private RepositoryVersionEntity findSpecifiedRepo(String repoVersionString, Long return repoVersion; } - public void createAmbariClusterResource(String clusterName, StackId stackId, SecurityType securityType) { - String stackInfo = stackId.toString(); + private void createAmbariClusterResource(String clusterName, Set stackIds, SecurityType securityType) { + String stackInfo = stackIds.iterator().next().toString(); // temporary final ClusterRequest clusterRequest = new ClusterRequest(null, clusterName, null, securityType, stackInfo, null); try { - RetryHelper.executeWithRetry(new Callable() { - @Override - public Object call() throws Exception { - getController().createCluster(clusterRequest); - return null; - } + RetryHelper.executeWithRetry(() -> { + getController().createCluster(clusterRequest); + return null; }); + addDefaultClusterSettings(clusterName); } catch (AmbariException e) { LOG.error("Failed to create Cluster resource: ", e); if (e.getCause() instanceof DuplicateResourceException) { @@ -342,37 +350,31 @@ public Object call() throws Exception { } } - public void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { - Set serviceGroups = Sets.newHashSet(DEFAULT_SERVICE_GROUP_NAME); - Collection services = topology.getBlueprint().getServices(); - - try { - Cluster cluster = getController().getClusters().getCluster(clusterName); - serviceGroups.removeAll(cluster.getServiceGroups().keySet()); - services.removeAll(cluster.getServices().keySet()); - } catch (AmbariException e) { - throw new RuntimeException("Failed to persist service and component resources: " + e, e); + // FIXME temporarily add default cluster settings -- should be provided by ClusterImpl itself + private void addDefaultClusterSettings(String clusterName) throws AmbariException { + Cluster cluster = getController().getClusters().getCluster(clusterName); + for (PropertyInfo p : getController().getAmbariMetaInfo().getClusterProperties()) { + cluster.addClusterSetting(p.getName(), p.getValue()); } + } - Set serviceGroupRequests = serviceGroups.stream() - .map(serviceGroupName -> new ServiceGroupRequest(clusterName, serviceGroupName, Iterables.getFirst(topology.getBlueprint().getStackIds(), null).getStackId())) + private void createAmbariServiceAndComponentResources(ClusterTopology topology, String clusterName, Map repoVersionByStack) { + Set serviceGroupRequests = topology.getComponents() + .map(c -> new ServiceGroupRequest(clusterName, c.effectiveServiceGroupName(), c.stackId().getStackId())) .collect(toSet()); - Set serviceRequests = new HashSet<>(); - Set componentRequests = new HashSet<>(); - for (String service : services) { - String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service); - StackId stackId = Iterables.getOnlyElement(topology.getBlueprint().getStackIdsForService(service)); // FIXME temporarily assume each service is defined in only one mpack - Long repositoryVersionId = repoVersionByStack.get(stackId); - serviceRequests.add(new ServiceRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, service, - repositoryVersionId, null, credentialStoreEnabled, stackId - )); + Set serviceRequests = topology.getComponents() + .map(c -> new ServiceRequest( + clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.serviceType(), repoVersionByStack.get(c.stackId()), null, + topology.getSetting().getCredentialStoreEnabled(c.effectiveServiceName()), // FIXME settings by service type or name? + c.stackId() + )) + .collect(toSet()); - for (String component : topology.getBlueprint().getComponentNames(service)) { - String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component); - componentRequests.add(new ServiceComponentRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, null, recoveryEnabled)); - } - } + Set componentRequests = topology.getComponents() + .map(c -> new ServiceComponentRequest(clusterName, c.effectiveServiceGroupName(), c.effectiveServiceName(), c.componentName(), null, + topology.getSetting().getRecoveryEnabled(c.effectiveServiceName(), c.componentName()))) // FIXME settings by service type or name? + .collect(toSet()); try { if (!serviceGroupRequests.isEmpty()) { @@ -406,7 +408,7 @@ public void createAmbariServiceAndComponentResources(ClusterTopology topology, S } } - public void createAmbariHostResources(long clusterId, String hostName, Map> components) { + public void createAmbariHostResources(long clusterId, String hostName, Stream components) { Host host; try { host = getController().getClusters().getHost(hostName); @@ -416,7 +418,7 @@ public void createAmbariHostResources(long clusterId, String hostName, Map requests = new HashSet<>(); + final Set requests = components + .filter(component -> !component.componentName().equals(RootComponent.AMBARI_SERVER.name())) + .map(component -> new ServiceComponentHostRequest(clusterName, component.effectiveServiceGroupName(), component.effectiveServiceName(), component.componentName(), hostName, null)) + .collect(toSet()); - for (Map.Entry> entry : components.entrySet()) { - String service = entry.getKey(); - for (String component : entry.getValue()) { - //todo: handle this in a generic manner. These checks are all over the code - try { - if (cluster.getService(service) != null && !component.equals(RootComponent.AMBARI_SERVER.name())) { - requests.add(new ServiceComponentHostRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, hostName, null)); - } - } catch(AmbariException se) { - LOG.warn("Service already deleted from cluster: {}", service); - } - } - } try { - RetryHelper.executeWithRetry(new Callable() { - @Override - public Object call() throws Exception { - getController().createHostComponents(requests); - return null; - } + RetryHelper.executeWithRetry(() -> { + getController().createHostComponents(requests); + return null; }); } catch (AmbariException e) { LOG.error("Unable to create host component resource for host {}", hostName, e); @@ -491,7 +480,7 @@ public static void init(HostRoleCommandFactory factory) { } public void registerHostWithConfigGroup(final String hostName, final ClusterTopology topology, final String groupName) { - String qualifiedGroupName = getConfigurationGroupName(topology.getBlueprint().getName(), groupName); + String qualifiedGroupName = getConfigurationGroupName(topology.getBlueprintName(), groupName); Lock configGroupLock = configGroupCreateLock.get(qualifiedGroupName); @@ -760,7 +749,7 @@ private boolean addHostToExistingConfigGroups(String hostName, ClusterTopology t */ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String groupName) throws AmbariException { Map> groupConfigs = new HashMap<>(); - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); // get the host-group config with cluster creation template overrides Configuration topologyHostGroupConfig = topology. @@ -774,7 +763,7 @@ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String for (Map.Entry> entry : userProvidedGroupProperties.entrySet()) { String type = entry.getKey(); String service = stack.getServicesForConfigType(type) - .filter(each -> topology.getBlueprint().getServices().contains(each)) + .filter(each -> topology.getServices().contains(each)) .findFirst() // TODO check if this is required at all (might be handled by the "orphan" removal) // TODO move this validation earlier @@ -790,7 +779,7 @@ private void createConfigGroupsAndRegisterHost(ClusterTopology topology, String serviceConfigs.put(type, config); } - String bpName = topology.getBlueprint().getName(); + String bpName = topology.getBlueprintName(); for (Map.Entry> entry : groupConfigs.entrySet()) { String service = entry.getKey(); Map serviceConfigs = entry.getValue(); @@ -834,6 +823,50 @@ public boolean apply(@Nullable String groupHost) { } } + public StackDefinition composeStacks(Set stackIds) { + Set stacks = stackIds.stream() + .map(this::createStack) + .collect(toSet()); + StackDefinition composite = StackDefinition.of(stacks); + + // temporary check + verifyStackDefinitionsAreDisjoint(composite.getServices().stream(), "Service", composite::getStacksForService); + verifyStackDefinitionsAreDisjoint(composite.getComponents().stream(), "Component", composite::getStacksForComponent); + + return composite; + } + + /** + * Verify that each item in items is defined by only one stack. + * + * @param items the items to check + * @param type string description of the type of items (eg. "Service", or "Component") + * @param lookup a function to find the set of stacks that an item belongs to + * @throws IllegalArgumentException if some items are defined in multiple stacks + */ + static void verifyStackDefinitionsAreDisjoint(Stream items, String type, Function> lookup) { + Set>> definedInMultipleStacks = items + .map(s -> Pair.of(s, lookup.apply(s))) + .filter(p -> p.getRight().size() > 1) + .collect(toCollection(TreeSet::new)); + + if (!definedInMultipleStacks.isEmpty()) { + String msg = definedInMultipleStacks.stream() + .map(p -> String.format("%s %s is defined in multiple stacks: %s", type, p.getLeft(), Joiner.on(", ").join(p.getRight()))) + .collect(joining("\n")); + LOG.error(msg); + throw new IllegalArgumentException(msg); + } + } + + protected Stack createStack(StackId stackId) { + try { + return stackFactory.createStack(stackId); + } catch (StackAccessException e) { + throw new NoSuchStackException(stackId); + } + } + /** * Get a config group name based on a bp and host group. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 04e004e5c55..64456247601 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -19,11 +19,11 @@ package org.apache.ambari.server.topology; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.ambari.server.controller.internal.StackDefinition; +import javax.annotation.Nonnull; + import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.state.StackId; @@ -73,80 +73,11 @@ public interface Blueprint { */ Setting getSetting(); - /** - * Get all of the services represented in the blueprint. - * - * @return collection of all represented service names - */ - Collection getServices(); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of components for the service. Will not return null. - */ - Collection getComponents(String service); - - /** - * Get the components that are included in the blueprint for the specified service. - * - * @param service service name - * - * @return collection of component names for the service. Will not return null. - */ - @Deprecated - Collection getComponentNames(String service); - - /** - * Get whether a component is enabled for auto start. - * - * @param serviceName - Service name. - * @param componentName - Component name. - * - * @return null if value is not specified; true or false if specified. - */ - String getRecoveryEnabled(String serviceName, String componentName); - - /** - * Get whether a service is enabled for credential store use. - * - * @param serviceName - Service name. - * - * @return null if value is not specified; true or false if specified. - */ - String getCredentialStoreEnabled(String serviceName); - - /** - * Check if auto skip failure is enabled. - * @return true if enabled, otherwise false. - */ - boolean shouldSkipFailure(); - - /** - * Get the stack associated with the blueprint. - * For mpack-based installation this is a composite stack - * that provides a single unified view of all underlying mpacks, - * but does not have any identifier. - * - * @return associated stack - */ - StackDefinition getStack(); - /** * @return the set of stack (mpack) IDs associated with the blueprint */ Set getStackIds(); - /** - * Look up the stacks that define service. - * To be used only after checking that services map to - * @param service the name of the service as defined in the stack (mpack), eg. ZOOKEEPER - * @return the ID of the stack that defines the given service - */ - Set getStackIdsForService(String service); - /** * Get the mpacks associated with the blueprint. * @@ -154,16 +85,6 @@ public interface Blueprint { */ Collection getMpacks(); - /** - * Get the host groups which contain components for the specified service. - * - * @param service service name - * - * @return collection of host groups containing components for the specified service; - * will not return null - */ - Collection getHostGroupsForService(String service); - /** * Get the host groups which contain the give component. * @@ -173,13 +94,9 @@ public interface Blueprint { */ Collection getHostGroupsForComponent(String component); + @Nonnull SecurityConfiguration getSecurity(); - /** - * A config type is valid if there are services related to except cluster-env and global. - */ - boolean isValidConfigType(String configType); - /** * Obtain the blueprint as an entity. * @@ -187,6 +104,9 @@ public interface Blueprint { */ BlueprintEntity toEntity(); - List getRepositorySettings(); + /** + * Add the kerberos client to all host groups in the blueprint. + */ + boolean ensureKerberosClientIsPresent(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java new file mode 100644 index 00000000000..43f372457ed --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintBasedClusterProvisionRequest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static java.util.stream.Collectors.toMap; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; + +import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.orm.entities.BlueprintEntity; +import org.apache.ambari.server.state.SecurityType; +import org.apache.ambari.server.state.StackId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +/** + * I am the Blueprint and ProvisionClusterRequest combined. + */ +public class BlueprintBasedClusterProvisionRequest implements Blueprint, ProvisionRequest { + + private static final Logger LOG = LoggerFactory.getLogger(BlueprintBasedClusterProvisionRequest.class); + + private final Blueprint blueprint; + private final ProvisionClusterRequest request; + private final Set stackIds; + private final StackDefinition stack; + private final Map mpacks; + private final SecurityConfiguration securityConfiguration; + + public BlueprintBasedClusterProvisionRequest(AmbariContext ambariContext, SecurityConfigurationFactory securityConfigurationFactory, Blueprint blueprint, ProvisionClusterRequest request) { + this.blueprint = blueprint; + this.request = request; + + stackIds = ImmutableSet.copyOf(Sets.union(blueprint.getStackIds(), request.getStackIds())); + stack = ambariContext.composeStacks(stackIds); + mpacks = ImmutableMap.copyOf( + Stream.concat(blueprint.getMpacks().stream(), request.getMpacks().stream()) + .collect(toMap(MpackInstance::getMpackName, Function.identity()))); + + securityConfiguration = processSecurityConfiguration(securityConfigurationFactory); + + if (securityConfiguration.getType() == SecurityType.KERBEROS) { + ensureKerberosClientIsPresent(); + } + } + + @Override + public String getName() { + return blueprint.getName(); + } + + @Override + public Map getHostGroups() { + return blueprint.getHostGroups(); + } + + @Override + public HostGroup getHostGroup(String name) { + return blueprint.getHostGroup(name); + } + + @Override + public Configuration getConfiguration() { + return request.getConfiguration(); + } + + @Override + public Setting getSetting() { + return blueprint.getSetting(); + } + + @Override + public Set getStackIds() { + return stackIds; + } + + @Override + public Collection getMpacks() { + return mpacks.values(); + } + + @Override + public Collection getHostGroupsForComponent(String component) { + return blueprint.getHostGroupsForComponent(component); + } + + @Nonnull + @Override + public SecurityConfiguration getSecurity() { + return securityConfiguration; + } + + @Override + public BlueprintEntity toEntity() { + throw new UnsupportedOperationException(); + } + + @Override + public Long getClusterId() { + return null; + } + + @Override + public Type getType() { + return Type.PROVISION; + } + + @Override + public Blueprint getBlueprint() { + return blueprint; + } + + @Override + public Map getHostGroupInfo() { + return request.getHostGroupInfo(); + } + + @Override + public String getDescription() { + return request.getDescription(); + } + + public String getDefaultPassword() { + return request.getDefaultPassword(); + } + + public ConfigRecommendationStrategy getConfigRecommendationStrategy() { + return request.getConfigRecommendationStrategy(); + } + + public ProvisionAction getProvisionAction() { + return request.getProvisionAction(); + } + + public StackDefinition getStack() { + return stack; + } + + public Map> getServicesByMpack() { + Map> result = new HashMap<>(); + for (MpackInstance mpack : mpacks.values()) { + Map services = mpack.getServiceInstances().stream() + .collect(toMap(ServiceInstance::getName, Function.identity())); + result.put(mpack.getMpackName(), services); + } + return result; + } + + /** + * @return service instances defined in the topology, mapped by service name, + * whose name is unique across all mpacks. + */ + public Map getUniqueServices() { + Map map = mpacks.values().stream() + .flatMap(mpack -> mpack.getServiceInstances().stream()) + .collect(toMap(ServiceInstance::getName, Function.identity(), (s1, s2) -> null)); + map.entrySet().removeIf(e -> e.getValue() == null); // remove non-unique names mapped to null + return map; + } + + /** + * Retrieve security info from Blueprint if missing from Cluster Template request. + */ + private SecurityConfiguration processSecurityConfiguration(SecurityConfigurationFactory securityConfigurationFactory) { + SecurityConfiguration blueprintSecurity = blueprint.getSecurity(); + SecurityConfiguration requestSecurity = request.getSecurityConfiguration(); + + if (requestSecurity == null) { + LOG.debug("There's no security configuration in the request, retrieving it from the associated blueprint"); + requestSecurity = blueprintSecurity; + if (requestSecurity.getType() == SecurityType.KERBEROS && requestSecurity.getDescriptorReference() != null) { + requestSecurity = securityConfigurationFactory.loadSecurityConfigurationByReference(requestSecurity.getDescriptorReference()); + } + } else if (requestSecurity.getType() == SecurityType.NONE && blueprintSecurity.getType() == SecurityType.KERBEROS) { + throw new IllegalArgumentException("Setting security to NONE is not allowed as security type in blueprint is set to KERBEROS!"); + } + + return requestSecurity; + } + + @Override + public boolean ensureKerberosClientIsPresent() { + return blueprint.ensureKerberosClientIsPresent(); + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 4386ec6919d..0ac76df322a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -40,22 +40,17 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Stream; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.ObjectNotFoundException; -import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.controller.AmbariServer; -import org.apache.ambari.server.controller.RootComponent; +import javax.inject.Inject; +import javax.inject.Provider; + import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.Stack; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.stack.NoSuchStackException; @@ -68,7 +63,6 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; -import com.google.inject.Inject; /** * Create a Blueprint instance. @@ -77,27 +71,21 @@ public class BlueprintFactory { private static final Logger LOG = LoggerFactory.getLogger(BlueprintFactory.class); - private static BlueprintDAO blueprintDAO; - + private final Provider blueprintDAO; private final ConfigurationFactory configFactory = new ConfigurationFactory(); - private final StackFactory stackFactory; - - public BlueprintFactory() { - this(new DefaultStackFactory()); - } - protected BlueprintFactory(StackFactory stackFactory) { - this.stackFactory = stackFactory; + @Inject + public BlueprintFactory(Provider blueprintDAO) { + this.blueprintDAO = blueprintDAO; } public Blueprint getBlueprint(String blueprintName) throws NoSuchStackException { - BlueprintEntity entity = blueprintDAO.findByName(blueprintName); + BlueprintEntity entity = blueprintDAO.get().findByName(blueprintName); if (entity != null) { Set stackIds = entity.getMpackInstances().stream() .map(m -> new StackId(m.getMpackName(), m.getMpackVersion())) .collect(toSet()); - StackDefinition stack = composeStacks(stackIds); - return new BlueprintImpl(entity, stack, stackIds); + return new BlueprintImpl(entity, stackIds); } return null; } @@ -120,22 +108,25 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu Collection mpackInstances = createMpackInstances(properties); if (mpackInstances.isEmpty()) { - StackId stackId = getStackId(properties); - mpackInstances = Collections.singleton(new MpackInstance(stackId.getStackName(), stackId.getStackVersion(), null, null, Configuration.createEmpty())); + Optional stackId = getStackId(properties); + if (stackId.isPresent()) { + String stackName = stackId.get().getStackName(); + String stackVersion = stackId.get().getStackVersion(); + mpackInstances = Collections.singleton(new MpackInstance(stackName, stackVersion, null, null, Configuration.createEmpty())); + } } Set stackIds = mpackInstances.stream() - .map(m -> new StackId(m.getMpackName(), m.getMpackVersion())) + .map(MpackInstance::getStackId) .collect(toSet()); - StackDefinition stack = composeStacks(stackIds); - Collection hostGroups = processHostGroups(name, stack, properties); + Collection hostGroups = processHostGroups(properties); Configuration configuration = configFactory.getConfiguration((Collection>) properties.get(CONFIGURATION_PROPERTY_ID)); Setting setting = SettingFactory.getSetting((Collection>) properties.get(SETTING_PROPERTY_ID)); - return new BlueprintImpl(name, hostGroups, stack, stackIds, mpackInstances, configuration, securityConfiguration, setting); + return new BlueprintImpl(name, hostGroups, stackIds, mpackInstances, configuration, securityConfiguration, setting); } - private Collection createMpackInstances(Map properties) throws NoSuchStackException { + public static Collection createMpackInstances(Map properties) throws NoSuchStackException { if (properties.containsKey(MPACK_INSTANCES_PROPERTY_ID)) { ObjectMapper mapper = new ObjectMapper(); mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); @@ -151,23 +142,12 @@ private Collection createMpackInstances(Map prope } } - private static StackId getStackId(Map properties) throws NoSuchStackException { - String stackName = String.valueOf(properties.get(STACK_NAME_PROPERTY_ID)); - String stackVersion = String.valueOf(properties.get(STACK_VERSION_PROPERTY_ID)); - return new StackId(stackName, stackVersion); - } - - private StackDefinition composeStacks(Set stackIds) { - Set stacks = stackIds.stream() - .map(this::createStack) - .collect(toSet()); - StackDefinition composite = StackDefinition.of(stacks); - - // temporary check - verifyStackDefinitionsAreDisjoint(composite.getServices().stream(), "Service", composite::getStacksForService); - verifyStackDefinitionsAreDisjoint(composite.getComponents().stream(), "Component", composite::getStacksForComponent); - - return composite; + private static Optional getStackId(Map properties) throws NoSuchStackException { + Object stackName = properties.get(STACK_NAME_PROPERTY_ID); + Object stackVersion = properties.get(STACK_VERSION_PROPERTY_ID); + return stackName != null && stackVersion != null + ? Optional.of(new StackId(stackName.toString(), stackVersion.toString())) + : Optional.empty(); } /** @@ -193,23 +173,10 @@ static void verifyStackDefinitionsAreDisjoint(Stream items, String type, } } - protected Stack createStack(StackId stackId) { - try { - //todo: don't pass in controller - return stackFactory.createStack(stackId, AmbariServer.getController()); - } catch (ObjectNotFoundException e) { - throw new NoSuchStackException(stackId); - } catch (AmbariException e) { - // todo - throw new RuntimeException( - String.format("An error occurred parsing the stack information for %s", stackId) , e); - } - } - //todo: Move logic to HostGroupImpl @SuppressWarnings("unchecked") - private Collection processHostGroups(String bpName, StackDefinition stack, Map properties) { - Set> hostGroupProps = (HashSet>) + private Collection processHostGroups(Map properties) { + Set> hostGroupProps = (Set>) properties.get(HOST_GROUP_PROPERTY_ID); if (hostGroupProps == null || hostGroupProps.isEmpty()) { @@ -217,49 +184,43 @@ private Collection processHostGroups(String bpName, StackDefinition s } Collection hostGroups = new ArrayList<>(); - for (HashMap hostGroupProperties : hostGroupProps) { + for (Map hostGroupProperties : hostGroupProps) { String hostGroupName = (String) hostGroupProperties.get(HOST_GROUP_NAME_PROPERTY_ID); if (hostGroupName == null || hostGroupName.isEmpty()) { throw new IllegalArgumentException("Every host group must include a non-null 'name' property"); } - HashSet> componentProps = (HashSet>) + Set> componentProps = (Set>) hostGroupProperties.get(COMPONENT_PROPERTY_ID); Collection> configProps = (Collection>) hostGroupProperties.get(CONFIGURATION_PROPERTY_ID); - Collection components = processHostGroupComponents(stack, hostGroupName, componentProps); + Collection components = processHostGroupComponents(hostGroupName, componentProps); Configuration configuration = configFactory.getConfiguration(configProps); String cardinality = String.valueOf(hostGroupProperties.get(HOST_GROUP_CARDINALITY_PROPERTY_ID)); - HostGroup group = new HostGroupImpl(hostGroupName, bpName, stack, components, configuration, cardinality); + HostGroup group = new HostGroupImpl(hostGroupName, components, configuration, cardinality); hostGroups.add(group); } return hostGroups; } - private Collection processHostGroupComponents(StackDefinition stack, String groupName, HashSet> componentProps) { + private Collection processHostGroupComponents(String groupName, Set> componentProps) { if (componentProps == null || componentProps.isEmpty()) { throw new IllegalArgumentException("Host group '" + groupName + "' must contain at least one component"); } - Collection stackComponentNames = getAllStackComponents(stack); Collection components = new ArrayList<>(); - for (HashMap componentProperties : componentProps) { + for (Map componentProperties : componentProps) { String componentName = componentProperties.get(COMPONENT_NAME_PROPERTY_ID); if (componentName == null || componentName.isEmpty()) { throw new IllegalArgumentException("Host group '" + groupName + "' contains a component with no 'name' property"); } - if (! stackComponentNames.contains(componentName)) { - throw new IllegalArgumentException("The component '" + componentName + "' in host group '" + - groupName + "' is not valid for the specified stack"); - } - String mpackInstance = componentProperties.get(COMPONENT_MPACK_INSTANCE_PROPERTY); String serviceInstance = componentProperties.get(COMPONENT_SERVICE_INSTANCE_PROPERTY); //TODO, might want to add some validation here, to only accept value enum types, rwn @@ -271,52 +232,4 @@ private Collection processHostGroupComponents(StackDefinition stack, return components; } - /** - * Obtain all component names for the specified stack. - * - * @return collection of component names for the specified stack - * @throws IllegalArgumentException if the specified stack doesn't exist - */ - private Collection getAllStackComponents(StackDefinition stack) { - Collection allComponents = new HashSet<>(stack.getComponents()); - - // currently ambari server is not a recognized component - allComponents.add(RootComponent.AMBARI_SERVER.name()); - - return allComponents; - } - - - /** - * Static initialization. - * - * @param dao blueprint data access object - */ - @Inject - public static void init(BlueprintDAO dao) { - blueprintDAO = dao; - } - - /** - * Internal interface used to abstract out the process of creating the Stack object. - * - * This is used to simplify unit testing, since a new Factory can be provided to - * simulate various Stack or error conditions. - */ - interface StackFactory { - Stack createStack(StackId stackId, AmbariManagementController managementController) throws AmbariException; - } - - /** - * Default implementation of StackFactory. - * - * Calls the Stack constructor to create the Stack instance. - * - */ - private static class DefaultStackFactory implements StackFactory { - @Override - public Stack createStack(StackId stackId, AmbariManagementController managementController) throws AmbariException { - return new Stack(stackId.getStackName(), stackId.getStackVersion(), managementController); - } - } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 04616a72270..47e19bafbe7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -19,8 +19,6 @@ package org.apache.ambari.server.topology; -import static java.util.stream.Collectors.toList; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -32,7 +30,6 @@ import java.util.Set; import java.util.function.Supplier; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintConfiguration; import org.apache.ambari.server.orm.entities.BlueprintEntity; @@ -43,14 +40,13 @@ import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; import org.apache.ambari.server.stack.NoSuchStackException; -import org.apache.ambari.server.state.ConfigHelper; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.utils.JsonUtils; -import org.apache.commons.lang.StringUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; /** @@ -61,37 +57,32 @@ public class BlueprintImpl implements Blueprint { private final String name; private final Map hostGroups; private final Collection mpacks; - private final StackDefinition stack; private final Set stackIds; private final Configuration configuration; private final SecurityConfiguration security; private final Setting setting; private final List repoSettings; - public BlueprintImpl(BlueprintEntity entity, StackDefinition stack, Set stackIds) throws NoSuchStackException { + public BlueprintImpl(BlueprintEntity entity, Set stackIds) throws NoSuchStackException { name = entity.getBlueprintName(); security = entity.getSecurityType() != null ? new SecurityConfiguration(entity.getSecurityType(), entity.getSecurityDescriptorReference(), null) : SecurityConfiguration.NONE; mpacks = parseMpacks(entity); - this.stack = stack; this.stackIds = stackIds; // create config first because it is set as a parent on all host-group configs configuration = processConfiguration(entity.getConfigurations()); hostGroups = parseBlueprintHostGroups(entity); - // TODO: how to handle multiple stacks correctly? - configuration.setParentConfiguration(stack.getConfiguration(getServices())); - setting = processSetting(entity.getSettings()); + setting = new Setting(parseSetting(entity.getSettings())); repoSettings = processRepoSettings(); } - public BlueprintImpl(String name, Collection groups, StackDefinition stack, Set stackIds, Collection mpacks, + public BlueprintImpl(String name, Collection groups, Set stackIds, Collection mpacks, Configuration configuration, SecurityConfiguration security, Setting setting) { this.name = name; this.mpacks = mpacks; - this.stack = stack; this.stackIds = stackIds; this.security = security != null ? security : SecurityConfiguration.NONE; @@ -100,29 +91,22 @@ public BlueprintImpl(String name, Collection groups, StackDefinition for (HostGroup hostGroup : groups) { hostGroups.put(hostGroup.getName(), hostGroup); } - // TODO: handle configuration from multiple stacks properly - // if the parent isn't set, the stack configuration is set as the parent this.configuration = configuration; - if (configuration.getParentConfiguration() == null) { - configuration.setParentConfiguration(stack.getConfiguration(getServices())); - } - this.setting = setting; + this.setting = setting != null ? setting : new Setting(ImmutableMap.of()); repoSettings = processRepoSettings(); } + @Override public String getName() { return name; } + @Override public Set getStackIds() { return stackIds; } @Override - public Set getStackIdsForService(String service) { - return stack.getStacksForService(service); - } - public SecurityConfiguration getSecurity() { return security; } @@ -149,162 +133,10 @@ public Setting getSetting() { return setting; } - /** - * Get all services represented in blueprint. - * - * @return collections of all services provided by topology - */ - @Override - public Collection getServices() { - Collection services = new HashSet<>(); - for (HostGroup group : getHostGroups().values()) { - services.addAll(group.getServices()); - } - return services; - } - - @Override - public Collection getComponents(String service) { - Collection components = new HashSet<>(); - for (HostGroup group : getHostGroupsForService(service)) { - components.addAll(group.getComponents(service)); - } - return components; - } - - @Override - @Deprecated - public Collection getComponentNames(String service) { - return getComponents(service).stream().map(Component::getName).collect(toList()); - } - - /** - * Get whether the specified component in the service is enabled - * for auto start. - * - * @param serviceName - Service name. - * @param componentName - Component name. - * - * @return null if value is not specified; true or false if specified. - */ - @Override - public String getRecoveryEnabled(String serviceName, String componentName) { - Set> settingValue; - - if (setting == null) - return null; - - // If component name was specified in the list of "component_settings", - // determine if recovery_enabled is true or false and return it. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_COMPONENT_SETTINGS); - for (Map setting : settingValue) { - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, componentName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - } - - // If component name is not specified, look up it's service. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS); - for ( Map setting : settingValue){ - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, serviceName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - } - - // If service name is not specified, look up the cluster setting. - settingValue = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); - for (Map setting : settingValue) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED))) { - return setting.get(Setting.SETTING_NAME_RECOVERY_ENABLED); - } - } - - return null; - } - - /** - * Get whether the specified service is enabled for credential store use. - * - *
-   *     {@code
-   *       {
-   *         "service_settings" : [
-   *         { "name" : "RANGER",
-   *           "recovery_enabled" : "true",
-   *           "credential_store_enabled" : "true"
-   *         },
-   *         { "name" : "HIVE",
-   *           "recovery_enabled" : "true",
-   *           "credential_store_enabled" : "false"
-   *         },
-   *         { "name" : "TEZ",
-   *           "recovery_enabled" : "false"
-   *         }
-   *       ]
-   *     }
-   *   }
-   * 
- * - * @param serviceName - Service name. - * - * @return null if value is not specified; true or false if specified. - */ - @Override - public String getCredentialStoreEnabled(String serviceName) { - if (setting == null) - return null; - - // Look up the service and return the credential_store_enabled value. - Set> settingValue = setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS); - for (Map setting : settingValue) { - String name = setting.get(Setting.SETTING_NAME_NAME); - if (StringUtils.equals(name, serviceName)) { - if (!StringUtils.isEmpty(setting.get(Setting.SETTING_NAME_CREDENTIAL_STORE_ENABLED))) { - return setting.get(Setting.SETTING_NAME_CREDENTIAL_STORE_ENABLED); - } - break; - } - } - - return null; - } - - @Override - public boolean shouldSkipFailure() { - if (setting == null) { - return false; - } - Set> settingValue = setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS); - for (Map setting : settingValue) { - if (setting.containsKey(Setting.SETTING_NAME_SKIP_FAILURE)) { - return setting.get(Setting.SETTING_NAME_SKIP_FAILURE).equalsIgnoreCase("true"); - } - } - return false; - } - - @Override public Collection getMpacks() { return mpacks; } - public StackDefinition getStack() { - return stack; - } - - /** - * Get host groups which contain a component. - * - * @param component component name - * - * @return collection of host groups which contain the specified component - */ @Override public Collection getHostGroupsForComponent(String component) { Collection resultGroups = new HashSet<>(); @@ -316,24 +148,7 @@ public Collection getHostGroupsForComponent(String component) { return resultGroups; } - /** - * Get host groups which contain a component for the given service. - * - * @param service service name - * - * @return collection of host groups which contain a component of the specified service - */ @Override - public Collection getHostGroupsForService(String service) { - Collection resultGroups = new HashSet<>(); - for (HostGroup group : hostGroups.values() ) { - if (group.getServices().contains(service)) { - resultGroups.add(group); - } - } - return resultGroups; - } - public BlueprintEntity toEntity() { BlueprintEntity entity = new BlueprintEntity(); entity.setBlueprintName(name); @@ -388,7 +203,7 @@ private Collection parseMpacks(BlueprintEntity blueprintEntity) t private Map parseBlueprintHostGroups(BlueprintEntity entity) { Map hostGroups = new HashMap<>(); for (HostGroupEntity hostGroupEntity : entity.getHostGroups()) { - HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), getStack()); + HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity); // set the bp configuration as the host group config parent hostGroup.getConfiguration().setParentConfiguration(configuration); hostGroups.put(hostGroupEntity.getName(), hostGroup); @@ -405,15 +220,6 @@ private Configuration processConfiguration(Collection blueprintSetting) { - return blueprintSetting != null - ? new Setting(parseSetting(blueprintSetting)) - : null; - } - /** * Obtain configuration as a map of config type to corresponding properties. * @@ -436,14 +242,16 @@ private Map> parseConfigurations(Collection>> parseSetting(Collection blueprintSetting) { + private static Map>> parseSetting(Collection blueprintSetting) { + if (blueprintSetting == null) { + return ImmutableMap.of(); + } - Map>> properties = new HashMap<>(); + Map>> properties = new HashMap<>(); Gson gson = new Gson(); for (BlueprintSettingEntity setting : blueprintSetting) { String settingName = setting.getSettingName(); - Set> settingProperties = gson.>>fromJson( - setting.getSettingData(), Set.class); + Set> settingProperties = gson.>>fromJson(setting.getSettingData(), Set.class); properties.put(settingName, settingProperties); } return properties; @@ -606,9 +414,9 @@ private void createBlueprintSettingEntities(BlueprintEntity blueprintEntity) { Setting blueprintSetting = getSetting(); if (blueprintSetting != null) { Map settingEntityMap = new HashMap<>(); - for (Map.Entry>> propEntry : blueprintSetting.getProperties().entrySet()) { + for (Map.Entry>> propEntry : blueprintSetting.getProperties().entrySet()) { String settingName = propEntry.getKey(); - Set> properties = propEntry.getValue(); + Set> properties = propEntry.getValue(); BlueprintSettingEntity settingEntity = new BlueprintSettingEntity(); settingEntityMap.put(settingName, settingEntity); @@ -622,45 +430,21 @@ private void createBlueprintSettingEntities(BlueprintEntity blueprintEntity) { } /** - * A config type is valid if there are services related to except cluster-env and global. + * Parse stack repo info stored in the blueprint_settings table */ - public boolean isValidConfigType(String configType) { - if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) { - return true; - } - String service = getStack().getServiceForConfigType(configType); - return getServices().contains(service); + private List processRepoSettings() { + return setting != null ? setting.processRepoSettings() : Collections.emptyList(); } /** - * Parse stack repo info stored in the blueprint_settings table + * Add the Kerberos client to all host groups. */ - private List processRepoSettings() { - if (setting == null) { - return Collections.emptyList(); - } - - Set> repositorySettingsValue = setting.getSettingValue(Setting.SETTING_NAME_REPOSITORY_SETTINGS); - if (repositorySettingsValue == null) { - return Collections.emptyList(); + public boolean ensureKerberosClientIsPresent() { + boolean changed = false; + for (HostGroup group : getHostGroups().values()) { + changed |= group.addComponent(new Component("KERBEROS_CLIENT")); } - - return repositorySettingsValue.stream() - .map(this::parseRepositorySetting) - .collect(toList()); - } - - private RepositorySetting parseRepositorySetting(Map setting){ - RepositorySetting result = new RepositorySetting(); - result.setOperatingSystem(setting.get(RepositorySetting.OPERATING_SYSTEM)); - result.setOverrideStrategy(setting.get(RepositorySetting.OVERRIDE_STRATEGY)); - result.setRepoId(setting.get(RepositorySetting.REPO_ID)); - result.setBaseUrl(setting.get(RepositorySetting.BASE_URL)); - return result; - } - - public List getRepositorySettings(){ - return repoSettings; + return changed; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index ee626bce84a..ae4fe6b6706 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -18,6 +18,10 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; + import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -75,8 +79,7 @@ public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor) { this.ambariContext = ambariContext; this.clusterTopology = clusterTopology; - Blueprint blueprint = clusterTopology.getBlueprint(); - this.stack = blueprint.getStack(); + this.stack = clusterTopology.getStack(); // set initial configuration (not topology resolved) this.configurationProcessor = new BlueprintConfigurationProcessor(clusterTopology); this.stackAdvisorBlueprintProcessor = stackAdvisorBlueprintProcessor; @@ -90,11 +93,9 @@ public ClusterConfigurationRequest(AmbariContext ambariContext, ClusterTopology * Remove config-types from the given configuration if there is no any services related to them (except cluster-env and global). */ private void removeOrphanConfigTypes(Configuration configuration) { - Blueprint blueprint = clusterTopology.getBlueprint(); - Collection configTypes = configuration.getAllConfigTypes(); for (String configType : configTypes) { - if (!blueprint.isValidConfigType(configType)) { + if (!clusterTopology.isValidConfigType(configType)) { configuration.removeConfigType(configType); LOG.info("Removing config type '{}' as related service is not present in either Blueprint or cluster creation template.", configType); } @@ -164,24 +165,24 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> stackDefaultProps = stackDefaults.getProperties(); // add clusterHostInfo containing components to hosts map, based on Topology, to use this one instead of // StageUtils.getClusterInfo() - Map componentHostsMap = createComponentHostMap(blueprint); + Map componentHostsMap = createComponentHostMap(); existingConfigurations.put("clusterHostInfo", componentHostsMap); try { // generate principals & keytabs for headless identities AmbariContext.getController().getKerberosHelper() .ensureHeadlessIdentities(cluster, existingConfigurations, - new HashSet<>(blueprint.getServices())); + new HashSet<>(clusterTopology.getServices())); // apply Kerberos specific configurations Map> updatedConfigs = AmbariContext.getController().getKerberosHelper() .getServiceConfigurationUpdates(cluster, existingConfigurations, - createServiceComponentMap(blueprint), null, null, true, false); + createServiceComponentMap(), null, null, true, false); // ****************************************************************************************** // Since Kerberos is being enabled, make sure the cluster-env/security_enabled property is @@ -198,7 +199,7 @@ private Set configureKerberos(Configuration clusterConfiguration, Map propertyMap = updatedConfigs.get(configType); Map clusterConfigProperties = existingConfigurations.get(configType); Map stackDefaultConfigProperties = stackDefaultProps.get(configType); @@ -228,24 +229,12 @@ private Set configureKerberos(Configuration clusterConfiguration, Map> createServiceComponentMap(Blueprint blueprint) { - Map> serviceComponents = new HashMap<>(); - Collection services = blueprint.getServices(); - - if(services != null) { - for (String service : services) { - Collection components = blueprint.getComponentNames(service); - serviceComponents.put(service, - (components == null) - ? Collections.emptySet() - : new HashSet<>(blueprint.getComponentNames(service))); - } - } - - return serviceComponents; + private Map> createServiceComponentMap() { + return clusterTopology.getComponents() + .collect(groupingBy(ResolvedComponent::effectiveServiceName, + mapping(ResolvedComponent::componentName, toSet()))); } /** @@ -278,19 +267,17 @@ private boolean propertyHasCustomValue(Map clusterConfigProperti return propertyHasCustomValue; } - private Map createComponentHostMap(Blueprint blueprint) { + private Map createComponentHostMap() { Map componentHostsMap = new HashMap<>(); - for (String service : blueprint.getServices()) { - Collection components = blueprint.getComponentNames(service); - for (String component : components) { - Collection componentHost = clusterTopology.getHostAssignmentsForComponent(component); - // retrieve corresponding clusterInfoKey for component using StageUtils - String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component); - if (clusterInfoKey == null) { - clusterInfoKey = component.toLowerCase() + "_hosts"; - } - componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); + for (ResolvedComponent component : clusterTopology.getComponents().collect(toSet())) { + String componentName = component.componentName(); + Collection componentHost = clusterTopology.getHostAssignmentsForComponent(componentName); + // retrieve corresponding clusterInfoKey for component using StageUtils + String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(componentName); + if (clusterInfoKey == null) { + clusterInfoKey = componentName.toLowerCase() + "_hosts"; } + componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); } return componentHostsMap; } @@ -300,7 +287,6 @@ private Collection getRequiredHostgroupsForKerberosConfiguration() { try { Cluster cluster = getCluster(); - Blueprint blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); Map> existingConfigurations = clusterConfiguration.getFullProperties(); @@ -309,7 +295,7 @@ private Collection getRequiredHostgroupsForKerberosConfiguration() { // apply Kerberos specific configurations Map> updatedConfigs = AmbariContext.getController().getKerberosHelper() .getServiceConfigurationUpdates(cluster, existingConfigurations, - createServiceComponentMap(blueprint), null, null, true, false); + createServiceComponentMap(), null, null, true, false); // retrieve hostgroup for component names extracted from variables like "{clusterHostInfo.(component_name) // _host}" @@ -352,10 +338,9 @@ public void setConfigurationsOnCluster(ClusterTopology clusterTopology, String t //todo: also handle setting of host group scoped configuration which is updated by config processor List configurationRequests = new LinkedList<>(); - Blueprint blueprint = clusterTopology.getBlueprint(); Configuration clusterConfiguration = clusterTopology.getConfiguration(); - for (String service : blueprint.getServices()) { + for (String service : clusterTopology.getServices()) { //todo: remove intermediate request type // one bp config request per service BlueprintServiceConfigRequest blueprintConfigRequest = new BlueprintServiceConfigRequest(service); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java index 69ccb617dae..67ee0070bb5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopology.java @@ -20,9 +20,15 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; import org.apache.ambari.server.controller.RequestStatusResponse; import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.StackId; /** * Represents a full cluster topology including all instance information as well as the associated @@ -38,18 +44,33 @@ public interface ClusterTopology { Long getClusterId(); /** - * Set the id of the cluster. + * Get the blueprint associated with the cluster. * - * @param clusterId cluster id + * @return associated blueprint */ - void setClusterId(Long clusterId); + Blueprint getBlueprint(); /** - * Get the blueprint associated with the cluster. + * Get the name of the blueprint associated with the cluster. * - * @return assocaited blueprint + * @return associated blueprint's name */ - Blueprint getBlueprint(); + String getBlueprintName(); + + /** + * Get the stack associated with the blueprint. + * For mpack-based installation this is a composite stack + * that provides a single unified view of all underlying mpacks, + * but does not have any identifier. + * + * @return associated stack + */ + StackDefinition getStack(); + + /** + * @return the set of stack (mpack) IDs associated with the cluster + */ + Set getStackIds(); /** * Get the cluster scoped configuration for the cluster. @@ -60,6 +81,15 @@ public interface ClusterTopology { */ Configuration getConfiguration(); + /** + * Get the Blueprint cluster scoped setting. + * The blueprint cluster scoped setting has the setting properties + * with the setting names associated with the blueprint. + * + * @return blueprint cluster scoped setting + */ + Setting getSetting(); + /** * Get host group information. * @@ -74,6 +104,7 @@ public interface ClusterTopology { * * @return collection of host group names which contain the specified component */ + @Deprecated // 1. component name is not enough, 2. only used for stack-specific checks/updates Collection getHostGroupsForComponent(String component); /** @@ -94,8 +125,37 @@ public interface ClusterTopology { * * @return collection of hosts for the specified component; will not return null */ + @Deprecated Collection getHostAssignmentsForComponent(String component); + /** + * Get all of the services represented in the blueprint. + * + * @return collection of all represented service names + */ + Collection getServices(); + + /** + * Get all of the components represented in the blueprint. + * + * @return collection of all represented components + */ + Stream getComponents(); + + /** + * Get the components that are included in the specified host group. + * + * @param hostGroup host group name + * @return stream of components for the service + */ + @Nonnull + Stream getComponentsInHostGroup(String hostGroup); + + /** + * A config type is valid if there are services related to except cluster-env and global. + */ + boolean isValidConfigType(String configType); + /** * Update the existing topology based on the provided topology request. * @@ -117,20 +177,6 @@ public interface ClusterTopology { */ void addHostToTopology(String hostGroupName, String host) throws InvalidTopologyException, NoSuchHostGroupException; - /** - * Determine if NameNode HA is enabled. - * - * @return true if NameNode HA is enabled; false otherwise - */ - boolean isNameNodeHAEnabled(); - - /** - * Determine if Yarn ResourceManager HA is enabled. - * - * @return true if Yarn ResourceManager HA is enabled; false otherwise - */ - boolean isYarnResourceManagerHAEnabled(); - /** * Determine if the cluster is kerberos enabled. * @@ -155,16 +201,8 @@ public interface ClusterTopology { */ RequestStatusResponse startHost(String hostName, boolean skipFailure); - void setConfigRecommendationStrategy(ConfigRecommendationStrategy strategy); - ConfigRecommendationStrategy getConfigRecommendationStrategy(); - /** - * Set request provision action : INSTALL vs INSTALL_AND_START - * @param provisionAction @ProvisionAction - */ - void setProvisionAction(ProvisionAction provisionAction); - ProvisionAction getProvisionAction(); Map getAdvisedConfigurations(); @@ -180,4 +218,12 @@ public interface ClusterTopology { String getDefaultPassword(); + /** + * Determine if the host group contains a master component. + * + * @return true if the host group contains a master component; false otherwise + */ + boolean containsMasterComponent(String hostGroup); + + Collection getHostGroups(); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java index f50e60f428d..95814579f19 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterTopologyImpl.java @@ -19,6 +19,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; @@ -28,55 +29,84 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.internal.BaseClusterRequest; +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.StackId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + /** * Represents a cluster topology. * Topology includes the the associated blueprint, cluster configuration and hostgroup -> host mapping. */ public class ClusterTopologyImpl implements ClusterTopology { - private Long clusterId; + private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class); - //todo: currently topology is only associated with a single bp - //todo: this will need to change to allow usage of multiple bp's for the same cluster - //todo: for example: provision using bp1 and scale using bp2 - private Blueprint blueprint; - private Configuration configuration; - private ConfigRecommendationStrategy configRecommendationStrategy; - private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START; - private Map advisedConfigurations = new HashMap<>(); + private final Set stackIds; + private final StackDefinition stack; + private Long clusterId; + private final Blueprint blueprint; + private final Configuration configuration; + private final ConfigRecommendationStrategy configRecommendationStrategy; + private final ProvisionAction provisionAction; + private final Map advisedConfigurations = new HashMap<>(); private final Map hostGroupInfoMap = new HashMap<>(); private final AmbariContext ambariContext; + private final BlueprintBasedClusterProvisionRequest provisionRequest; private final String defaultPassword; + private final Map> resolvedComponents; - private final static Logger LOG = LoggerFactory.getLogger(ClusterTopologyImpl.class); - - - //todo: will need to convert all usages of hostgroup name to use fully qualified name (BP/HG) - //todo: for now, restrict scaling to the same BP public ClusterTopologyImpl(AmbariContext ambariContext, TopologyRequest topologyRequest) throws InvalidTopologyException { + this.ambariContext = ambariContext; this.clusterId = topologyRequest.getClusterId(); - // provision cluster currently requires that all hostgroups have same BP so it is ok to use root level BP here this.blueprint = topologyRequest.getBlueprint(); this.configuration = topologyRequest.getConfiguration(); - if (topologyRequest instanceof ProvisionClusterRequest) { - this.defaultPassword = ((ProvisionClusterRequest) topologyRequest).getDefaultPassword(); - } else { - this.defaultPassword = null; - } + configRecommendationStrategy = ConfigRecommendationStrategy.NEVER_APPLY; + provisionAction = topologyRequest instanceof BaseClusterRequest ? ((BaseClusterRequest) topologyRequest).getProvisionAction() : INSTALL_AND_START; // FIXME + + provisionRequest = null; + defaultPassword = null; + stackIds = topologyRequest.getBlueprint().getStackIds(); + stack = ambariContext.composeStacks(stackIds); + resolvedComponents = ImmutableMap.of(); registerHostGroupInfo(topologyRequest.getHostGroupInfo()); + } - // todo extract validation to specialized service - validateTopology(); + // FIXME 2. replayed request should simply be a provision or scale request + // FIXME 3. do not create a ClusterTopologyImpl for scale request -- create for original provision request only + public ClusterTopologyImpl( + AmbariContext ambariContext, + BlueprintBasedClusterProvisionRequest request, + Map> resolvedComponents + ) throws InvalidTopologyException { this.ambariContext = ambariContext; + this.blueprint = request.getBlueprint(); + this.configuration = request.getConfiguration(); + this.provisionRequest = request; + this.resolvedComponents = resolvedComponents; + configRecommendationStrategy = request.getConfigRecommendationStrategy(); + provisionAction = request.getProvisionAction(); + + defaultPassword = provisionRequest.getDefaultPassword(); + stackIds = request.getStackIds(); + stack = request.getStack(); + + blueprint.getConfiguration().setParentConfiguration(stack.getConfiguration(getServices())); + registerHostGroupInfo(request.getHostGroupInfo()); } @Override @@ -89,7 +119,6 @@ public Long getClusterId() { return clusterId; } - @Override public void setClusterId(Long clusterId) { this.clusterId = clusterId; } @@ -99,30 +128,52 @@ public Blueprint getBlueprint() { return blueprint; } + @Override + public String getBlueprintName() { + return blueprint.getName(); + } + + @Override + public Set getStackIds() { + return stackIds; + } + + @Override + public StackDefinition getStack() { + return stack; + } + @Override public Configuration getConfiguration() { return configuration; } + @Override + public Setting getSetting() { + return provisionRequest.getSetting(); + } + @Override public Map getHostGroupInfo() { return hostGroupInfoMap; } - //todo: do we want to return groups with no requested hosts? + @Override + public Collection getHostGroups() { + return blueprint.getHostGroups().values(); + } + @Override public Collection getHostGroupsForComponent(String component) { - Collection resultGroups = new ArrayList<>(); - for (HostGroup group : getBlueprint().getHostGroups().values() ) { - if (group.getComponentNames().contains(component)) { - resultGroups.add(group.getName()); - } - } - return resultGroups; + return resolvedComponents.entrySet().stream() + .filter(e -> e.getValue().stream().anyMatch(c -> component.equals(c.componentName()))) + .map(Map.Entry::getKey) + .collect(toSet()); } @Override public String getHostGroupForHost(String hostname) { + // FIXME change to map lookup for (HostGroupInfo groupInfo : hostGroupInfoMap.values() ) { if (groupInfo.getHostNames().contains(hostname)) { // a host can only be associated with a single host group @@ -178,36 +229,47 @@ public Collection getHostAssignmentsForComponent(String component) { } @Override - public boolean isNameNodeHAEnabled() { - return isNameNodeHAEnabled(configuration.getFullProperties()); + public Collection getServices() { + return getComponents() + .map(ResolvedComponent::effectiveServiceName) + .collect(toSet()); + } + + @Override + public Stream getComponents() { + return resolvedComponents.values().stream() + .flatMap(Collection::stream); } - public static boolean isNameNodeHAEnabled(Map> configurationProperties) { - return configurationProperties.containsKey("hdfs-site") && - (configurationProperties.get("hdfs-site").containsKey("dfs.nameservices") || - configurationProperties.get("hdfs-site").containsKey("dfs.internal.nameservices")); + @Override @Nonnull + public Stream getComponentsInHostGroup(String hostGroup) { + return resolvedComponents.computeIfAbsent(hostGroup, __ -> ImmutableSet.of()).stream(); } @Override - public boolean isYarnResourceManagerHAEnabled() { - return isYarnResourceManagerHAEnabled(configuration.getFullProperties()); + public boolean containsMasterComponent(String hostGroup) { + return resolvedComponents.getOrDefault(hostGroup, ImmutableSet.of()).stream() + .anyMatch(ResolvedComponent::masterComponent); } - /** - * Static convenience function to determine if Yarn ResourceManager HA is enabled - * @param configProperties configuration properties for this cluster - * @return true if Yarn ResourceManager HA is enabled - * false if Yarn ResourceManager HA is not enabled - */ - static boolean isYarnResourceManagerHAEnabled(Map> configProperties) { - return configProperties.containsKey("yarn-site") && configProperties.get("yarn-site").containsKey("yarn.resourcemanager.ha.enabled") - && configProperties.get("yarn-site").get("yarn.resourcemanager.ha.enabled").equals("true"); + @Override + public boolean isValidConfigType(String configType) { + if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) { + return true; + } + try { + String service = getStack().getServiceForConfigType(configType); + return getServices().contains(service); + } catch (IllegalArgumentException e) { + return false; + } } + // FIXME move out private void validateTopology() throws InvalidTopologyException { - if(isNameNodeHAEnabled()){ + if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(getConfiguration().getFullProperties())) { Collection nnHosts = getHostAssignmentsForComponent("NAMENODE"); if (nnHosts.size() != 2) { throw new InvalidTopologyException("NAMENODE HA requires exactly 2 hosts running NAMENODE but there are: " + @@ -215,8 +277,8 @@ private void validateTopology() } Map hadoopEnvConfig = configuration.getFullProperties().get("hadoop-env"); if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - if((!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"))) - || (!HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))){ + if ((!BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_active"))) + || (!BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches() && !nnHosts.contains(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")))) { throw new IllegalArgumentException("NAMENODE HA hosts mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected hosts are: " + nnHosts); } } @@ -271,11 +333,6 @@ public RequestStatusResponse startHost(String hostName, boolean skipFailure) { } } - @Override - public void setConfigRecommendationStrategy(ConfigRecommendationStrategy strategy) { - this.configRecommendationStrategy = strategy; - } - @Override public ConfigRecommendationStrategy getConfigRecommendationStrategy() { return this.configRecommendationStrategy; @@ -286,11 +343,6 @@ public ProvisionAction getProvisionAction() { return provisionAction; } - @Override - public void setProvisionAction(ProvisionAction provisionAction) { - this.provisionAction = provisionAction; - } - @Override public Map getAdvisedConfigurations() { return this.advisedConfigurations; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java index a62d341d3cc..3cc9c9931e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java @@ -28,11 +28,8 @@ public class Component { private final String name; - @Nullable private final String mpackInstance; - @Nullable private final String serviceInstance; - private final ProvisionAction provisionAction; @Deprecated @@ -40,7 +37,7 @@ public Component(String name) { this(name, null, null, null); } - public Component(String name, String mpackInstance, String serviceInstance, ProvisionAction provisionAction) { + public Component(String name, @Nullable String mpackInstance, @Nullable String serviceInstance, ProvisionAction provisionAction) { this.name = name; this.mpackInstance = mpackInstance; this.serviceInstance = serviceInstance; @@ -57,14 +54,14 @@ public String getName() { } /** - * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambigous) + * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambiguous) */ public String getMpackInstance() { return mpackInstance; } /** - * @return the service instance this component belongs to. Can be {@null} if component does not belong to a service + * @return the service instance this component belongs to. Can be {@code null} if component does not belong to a service * instance (there is a single service of the component's service type) */ public String getServiceInstance() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java new file mode 100644 index 00000000000..1738d88b424 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentResolver.java @@ -0,0 +1,20 @@ +package org.apache.ambari.server.topology; + +import java.util.Map; +import java.util.Set; + +/** + * Resolves all incompletely specified host group components in the topology: + * finds stack and/or service type that each component is defined in. + */ +public interface ComponentResolver { + + /** + * @return the set resolved components for each host group (the map's keys are host group names) + * @throws IllegalArgumentException if the components cannot be unambiguously resolved + * (eg. if some component is not known, or if there are multiple component with the same name and + * the request does not specify which one to select) + */ + Map> resolveComponents(BlueprintBasedClusterProvisionRequest request); + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java new file mode 100644 index 00000000000..2eb1bd64aa9 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/DefaultStackFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import javax.inject.Inject; +import javax.inject.Provider; + +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.StackId; + +/** + * Default implementation of StackFactory. + * + * Calls the Stack constructor to create the Stack instance. + * + */ +public class DefaultStackFactory implements StackFactory { + + @Inject + private Provider metaInfo; + + @Override + public Stack createStack(StackId stackId) throws StackAccessException { + return new Stack(stackId, metaInfo.get()); + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java index b444d01345f..fcc052e559e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/GPLLicenseNotAcceptedException.java @@ -21,7 +21,7 @@ /** * Indicates an not permitted LZO usage. */ -public class GPLLicenseNotAcceptedException extends Exception { +public class GPLLicenseNotAcceptedException extends InvalidTopologyException { public GPLLicenseNotAcceptedException(String s) { super(s); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java index a329f42053f..9ece5a26e75 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -19,20 +19,14 @@ package org.apache.ambari.server.topology; import java.util.Collection; -import java.util.regex.Pattern; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.StackDefinition; /** * Host Group representation. */ public interface HostGroup { - /** - * Compiled regex for hostgroup token. - */ - Pattern HOSTGROUP_REGEX = Pattern.compile("%HOSTGROUP::(\\S+?)%"); /** * Get the name of the host group. * @@ -40,21 +34,6 @@ public interface HostGroup { */ String getName(); - /** - * Get the name of the associated blueprint - * - * @return associated blueprint name - */ - String getBlueprintName(); - - /** - * Get the fully qualified host group name in the form of - * blueprintName:hostgroupName - * - * @return fully qualified host group name - */ - String getFullyQualifiedName(); - /** * Get all of the host group components. * @@ -62,7 +41,6 @@ public interface HostGroup { */ Collection getComponents(); - /** * Get all of the host group component names * @@ -84,48 +62,11 @@ public interface HostGroup { @Deprecated Collection getComponentNames(ProvisionAction provisionAction); - /** - * Get the names components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of component names - */ - @Deprecated - public Collection getComponentNames(String service); - - /** - * Get the host group components which belong to the specified service. - * - * @param service service instance name or service name. First, services looked up - * by instance name. If no appropriate service instance is found, services are looked - * up by type - * - * @return collection of component names for the specified service; will not return null - */ - Collection getComponents(String service); - - - /** * Add a component to the host group */ boolean addComponent(Component component); - /** - * Determine if the host group contains a master component. - * - * @return true if the host group contains a master component; false otherwise - */ - boolean containsMasterComponent(); - - /** - * Get all of the services associated with the host group components. - * - * @return collection of service names - */ - Collection getServices(); - /** * Get the configuration associated with the host group. * The host group configuration has the blueprint cluster scoped @@ -135,13 +76,6 @@ public interface HostGroup { */ Configuration getConfiguration(); - /** - * Get the stack associated with the host group. - * - * @return associated stack - */ - StackDefinition getStack(); - /** * Get the cardinality value that was specified for the host group. * This is simply meta-data for the stack that a deployer can use diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java index 0899d0bf111..be607e7b0ba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java @@ -20,10 +20,8 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -31,12 +29,9 @@ import java.util.Set; import org.apache.ambari.server.controller.internal.ProvisionAction; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.google.gson.Gson; @@ -45,63 +40,38 @@ */ public class HostGroupImpl implements HostGroup { - private static final Logger LOG = LoggerFactory.getLogger(HostGroupImpl.class); - /** * host group name */ - private String name; - - /** - * blueprint name - */ - private String blueprintName; + private final String name; /** * components contained in the host group */ private final Set components = new LinkedHashSet<>(); - /** - * map of service to components for the host group - */ - // TODO: in blueprint 3.0 this should be per service instance - private Map> componentsForService = new HashMap<>(); - /** * configuration */ - private Configuration configuration = null; - - private boolean containsMasterComponent = false; // FIXME never set + private final Configuration configuration; + private final String cardinality; - private StackDefinition stack; - - private String cardinality = "NOT SPECIFIED"; - - public HostGroupImpl(HostGroupEntity entity, String blueprintName, StackDefinition stack) { + public HostGroupImpl(HostGroupEntity entity) { this.name = entity.getName(); this.cardinality = entity.getCardinality(); - this.blueprintName = blueprintName; - this.stack = stack; + configuration = parseConfigurations(entity); parseComponents(entity); - parseConfigurations(entity); } - public HostGroupImpl(String name, String bpName, StackDefinition stack, Collection components, Configuration configuration, String cardinality) { + public HostGroupImpl(String name, Collection components, Configuration configuration, String cardinality) { this.name = name; - this.blueprintName = bpName; - this.stack = stack; + this.configuration = configuration; + this.cardinality = (cardinality != null && !"null".equals(cardinality)) ? cardinality : "NOT SPECIFIED"; // process each component for (Component component: components) { addComponent(component); } - - this.configuration = configuration; - if (cardinality != null && ! cardinality.equals("null")) { - this.cardinality = cardinality; - } } @@ -110,17 +80,6 @@ public String getName() { return name; } - //todo: currently not qualifying host group name - @Override - public String getFullyQualifiedName() { - return String.format("%s:%s", blueprintName, getName()); - } - - //todo: currently not qualifying host group name - public static String formatAbsoluteName(String bpName, String hgName) { - return String.format("%s:%s", bpName, hgName); - } - @Override public Collection getComponents() { return components; @@ -144,68 +103,15 @@ public Collection getComponentNames(ProvisionAction provisionAction) { return setOfComponentNames; } - /** - * Get the services which are deployed to this host group. - * - * @return collection of services which have components in this host group - */ - @Override - public Collection getServices() { - return componentsForService.keySet(); - } - /** * Adds a component to the host group. * @param component the component to add */ @Override public boolean addComponent(Component component) { - if (components.add(component)) { - containsMasterComponent |= stack.isMasterComponent(component.getName()); - - String service = stack.getServiceForComponent(component.getName()); - if (service != null) { - componentsForService - .computeIfAbsent(service, __ -> new HashSet<>()) - .add(component); - } - - return true; - } - - return false; + return components.add(component); } - /** - * Get the components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of components - */ - @Override - public Collection getComponents(String service) { - return componentsForService.containsKey(service) ? - new HashSet<>(componentsForService.get(service)) : - Collections.emptySet(); - } - - /** - * Get the names of components for the specified service which are associated with the host group. - * - * @param service service name - * - * @return set of component names - */ - @Override - @Deprecated - public Collection getComponentNames(String service) { - return componentsForService.containsKey(service) ? - componentsForService.get(service).stream().map(Component::getName).collect(toSet()) : - Collections.emptySet(); - } - - /** * Get this host groups configuration. * @@ -216,26 +122,6 @@ public Configuration getConfiguration() { return configuration; } - /** - * Get the associated blueprint name. - * - * @return associated blueprint name - */ - @Override - public String getBlueprintName() { - return blueprintName; - } - - @Override - public boolean containsMasterComponent() { - return containsMasterComponent; - } - - @Override - public StackDefinition getStack() { - return stack; - } - @Override public String getCardinality() { return cardinality; @@ -259,16 +145,12 @@ private void parseComponents(HostGroupEntity entity) { * Parse host group configurations. */ //todo: use ConfigurationFactory - private void parseConfigurations(HostGroupEntity entity) { + private Configuration parseConfigurations(HostGroupEntity entity) { Map> config = new HashMap<>(); Gson jsonSerializer = new Gson(); for (HostGroupConfigEntity configEntity : entity.getConfigurations()) { String type = configEntity.getType(); - Map typeProperties = config.get(type); - if (typeProperties == null) { - typeProperties = new HashMap<>(); - config.put(type, typeProperties); - } + Map typeProperties = config.computeIfAbsent(type, k -> new HashMap<>()); Map propertyMap = jsonSerializer.>fromJson( configEntity.getConfigData(), Map.class); if (propertyMap != null) { @@ -277,9 +159,10 @@ private void parseConfigurations(HostGroupEntity entity) { } //todo: parse attributes Map>> attributes = new HashMap<>(); - configuration = new Configuration(config, attributes); + return new Configuration(config, attributes); } + @Override public String toString(){ return name; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java index 3a8acc82162..a8ecc12d2cf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostRequest.java @@ -98,7 +98,7 @@ public HostRequest(long requestId, long id, long clusterId, String hostname, Str this.hostGroup = hostGroup; hostgroupName = hostGroup.getName(); this.predicate = predicate; - containsMaster = hostGroup.containsMasterComponent(); + containsMaster = topology.containsMasterComponent(hostgroupName); this.topology = topology; this.skipFailure = skipFailure; createTasks(this.skipFailure); @@ -121,14 +121,14 @@ public HostRequest(long requestId, long id, String predicate, this.requestId = requestId; this.id = id; clusterId = topology.getClusterId(); - blueprint = topology.getBlueprint().getName(); + blueprint = topology.getBlueprintName(); hostgroupName = entity.getTopologyHostGroupEntity().getName(); hostGroup = topology.getBlueprint().getHostGroup(hostgroupName); hostname = entity.getHostName(); setStatus(entity.getStatus()); statusMessage = entity.getStatusMessage(); this.predicate = toPredicate(predicate); - containsMaster = hostGroup.containsMasterComponent(); + containsMaster = topology.containsMasterComponent(hostgroupName); this.topology = topology; this.skipFailure = skipFailure; @@ -257,7 +257,7 @@ private void createTasks(boolean skipFailure) { "PENDING HOST ASSIGNMENT : HOSTGROUP=" + getHostgroupName(); AmbariContext context = topology.getAmbariContext(); - StackDefinition stack = hostGroup.getStack(); + StackDefinition stack = topology.getStack(); // Skip INSTALL task in case server component is marked as START_ONLY, or the cluster provision_action is // START_ONLY, unless component is marked with INSTALL_ONLY or INSTALL_AND_START. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java index b9bbe2cf12c..39c711a426f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/LogicalRequest.java @@ -444,7 +444,7 @@ public Optional getFailureReason() { private void createHostRequests(TopologyRequest request, ClusterTopology topology) { Map hostGroupInfoMap = request.getHostGroupInfo(); Blueprint blueprint = topology.getBlueprint(); - boolean skipFailure = topology.getBlueprint().shouldSkipFailure(); + boolean skipFailure = topology.getSetting().shouldSkipFailure(); for (HostGroupInfo hostGroupInfo : hostGroupInfoMap.values()) { String groupName = hostGroupInfo.getHostGroupName(); int hostCardinality = hostGroupInfo.getRequestedHostCount(); @@ -455,14 +455,14 @@ private void createHostRequests(TopologyRequest request, ClusterTopology topolog // host names are specified String hostname = hostnames.get(i); HostRequest hostRequest = new HostRequest(getRequestId(), hostIdCounter.getAndIncrement(), getClusterId(), - hostname, blueprint.getName(), blueprint.getHostGroup(groupName), null, topology, skipFailure); + hostname, topology.getBlueprintName(), blueprint.getHostGroup(groupName), null, topology, skipFailure); synchronized (requestsWithReservedHosts) { requestsWithReservedHosts.put(hostname, hostRequest); } } else { // host count is specified HostRequest hostRequest = new HostRequest(getRequestId(), hostIdCounter.getAndIncrement(), getClusterId(), - null, blueprint.getName(), blueprint.getHostGroup(groupName), hostGroupInfo.getPredicate(), topology, skipFailure); + null, topology.getBlueprintName(), blueprint.getHostGroup(groupName), hostGroupInfo.getPredicate(), topology, skipFailure); outstandingHostRequests.add(hostRequest); } } @@ -495,7 +495,7 @@ private void createHostRequests(ClusterTopology topology, } } - boolean skipFailure = topology.getBlueprint().shouldSkipFailure(); + boolean skipFailure = topology.getSetting().shouldSkipFailure(); for (TopologyHostRequestEntity hostRequestEntity : requestEntity.getTopologyHostRequestEntities()) { Long hostRequestId = hostRequestEntity.getId(); synchronized (hostIdCounter) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java index 6b57b0020e7..eeb45b776eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java @@ -26,6 +26,7 @@ import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity; import org.apache.ambari.server.orm.entities.BlueprintServiceConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintServiceEntity; +import org.apache.ambari.server.state.StackId; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -71,8 +72,8 @@ public void setMpackVersion(String mpackVersion) { } @JsonIgnore - public String getMpackNameAndVersion() { - return mpackName + "-" + mpackVersion; + public StackId getStackId() { + return new StackId(getMpackName(), getMpackVersion()); } @JsonIgnore diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java index bb1f85234d7..13c9eff95e6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/PersistedStateImpl.java @@ -31,6 +31,7 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.api.predicate.InvalidQueryException; import org.apache.ambari.server.controller.internal.BaseClusterRequest; +import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.orm.dao.HostDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; import org.apache.ambari.server.orm.dao.TopologyHostGroupDAO; @@ -215,9 +216,6 @@ public Map> getAllRequests() { if (clusterTopology == null) { try { clusterTopology = new ClusterTopologyImpl(ambariContext, replayedRequest); - if (entity.getProvisionAction() != null) { - clusterTopology.setProvisionAction(entity.getProvisionAction()); - } topologyRequests.put(replayedRequest.getClusterId(), clusterTopology); allRequests.put(clusterTopology, new ArrayList<>()); } catch (InvalidTopologyException e) { @@ -392,11 +390,13 @@ private static class ReplayedTopologyRequest implements TopologyRequest { private final Blueprint blueprint; private final Configuration configuration; private final Map hostGroupInfoMap = new HashMap<>(); + private final ProvisionAction provisionAction; public ReplayedTopologyRequest(TopologyRequestEntity entity, BlueprintFactory blueprintFactory) { clusterId = entity.getClusterId(); type = Type.valueOf(entity.getAction()); description = entity.getDescription(); + provisionAction = entity.getProvisionAction(); try { blueprint = blueprintFactory.getBlueprint(entity.getBlueprintName()); @@ -451,6 +451,10 @@ private Configuration createConfiguration(String propString, String attributeStr return new Configuration(properties, attributes); } + public ProvisionAction getProvisionAction() { + return provisionAction; + } + private void parseHostGroupInfo(TopologyRequestEntity entity) { for (TopologyHostGroupEntity hostGroupEntity : entity.getTopologyHostGroupEntities()) { for (TopologyHostInfoEntity hostInfoEntity : hostGroupEntity.getTopologyHostInfoEntities()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java new file mode 100644 index 00000000000..46bad4ebde8 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionRequest.java @@ -0,0 +1,17 @@ +package org.apache.ambari.server.topology; + +import java.util.Collection; +import java.util.Set; + +import org.apache.ambari.server.controller.internal.ProvisionAction; +import org.apache.ambari.server.state.StackId; + +public interface ProvisionRequest extends TopologyRequest { + + ConfigRecommendationStrategy getConfigRecommendationStrategy(); + ProvisionAction getProvisionAction(); + String getDefaultPassword(); + Set getStackIds(); + Collection getMpacks(); + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java index 35e370e83d8..3e7571c84d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/RepositorySetting.java @@ -18,6 +18,8 @@ package org.apache.ambari.server.topology; +import java.util.Map; + public class RepositorySetting { /** * Settings for each repo setting sections @@ -34,6 +36,15 @@ public class RepositorySetting { private String repoId; private String baseUrl; + static RepositorySetting fromMap(Map setting) { + RepositorySetting result = new RepositorySetting(); + result.setOperatingSystem(setting.get(OPERATING_SYSTEM)); + result.setOverrideStrategy(setting.get(OVERRIDE_STRATEGY)); + result.setRepoId(setting.get(REPO_ID)); + result.setBaseUrl(setting.get(BASE_URL)); + return result; + } + /** * When specified under the "settings" section, it allows Ambari to overwrite existing repos stored * in the metainfo table in the Ambari server database. @@ -106,11 +117,10 @@ public void setBaseUrl(String baseUrl) { } public String toString(){ - StringBuilder strBldr = new StringBuilder(); - strBldr.append(OVERRIDE_STRATEGY);strBldr.append(": ");strBldr.append(overrideStrategy); - strBldr.append(OPERATING_SYSTEM);strBldr.append(": ");strBldr.append(operatingSystem); - strBldr.append(REPO_ID);strBldr.append(": ");strBldr.append(repoId); - strBldr.append(BASE_URL);strBldr.append(": ");strBldr.append(baseUrl); - return strBldr.toString(); + return + OVERRIDE_STRATEGY + ": " + overrideStrategy + + OPERATING_SYSTEM + ": " + operatingSystem + + REPO_ID + ": " + repoId + + BASE_URL + ": " + baseUrl; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java new file mode 100644 index 00000000000..85e524bcefe --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ResolvedComponent.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import java.util.Optional; + +import org.apache.ambari.server.state.StackId; +import org.inferred.freebuilder.FreeBuilder; + +/** + * I provide additional information for a component specified in the blueprint, + * based on values resolved from the stack and sensible defaults. + */ +@FreeBuilder +public interface ResolvedComponent { + + StackId stackId(); + Optional serviceGroupName(); + String serviceType(); + Optional serviceName(); + String componentName(); + boolean masterComponent(); + + /** + * @return the component as specified in the blueprint + */ + Component component(); + + /** + * @return service group name if it set, otherwise defaults to the stack name + */ + default String effectiveServiceGroupName() { + return serviceGroupName().orElseGet(() -> stackId().getStackName()); + } + + /** + * @return service name if it set, otherwise defaults to the service type (eg. ZOOKEEPER) + */ + default String effectiveServiceName() { + return serviceName().orElseGet(this::serviceType); + } + + /** + * Starts building a {@code ResolvedComponent} for the given component. + */ + static Builder builder(Component component) { + return new Builder() + .component(component) + .componentName(component.getName()) + .serviceName(Optional.ofNullable(component.getServiceInstance())) + .serviceGroupName(Optional.ofNullable(component.getMpackInstance())); + } + + class Builder extends ResolvedComponent_Builder { + protected Builder() { + masterComponent(false); + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java index 904c784dac3..cf134735605 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Setting.java @@ -18,32 +18,31 @@ package org.apache.ambari.server.topology; -import java.util.Collections; -import java.util.HashMap; +import static java.util.stream.Collectors.toList; + +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import org.apache.commons.lang.StringUtils; + +import com.google.common.collect.ImmutableSet; + public class Setting { /** * Settings for this configuration instance */ - private Map>> properties; - - public static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings"; - - public static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings"; - - public static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings"; - - public static final String SETTING_NAME_DEPLOYMENT_SETTINGS = "deployment_settings"; - - public static final String SETTING_NAME_RECOVERY_ENABLED = "recovery_enabled"; + private final Map>> properties; + static final String SETTING_NAME_RECOVERY_SETTINGS = "recovery_settings"; + static final String SETTING_NAME_SERVICE_SETTINGS = "service_settings"; + static final String SETTING_NAME_COMPONENT_SETTINGS = "component_settings"; + static final String SETTING_NAME_DEPLOYMENT_SETTINGS = "deployment_settings"; + static final String SETTING_NAME_RECOVERY_ENABLED = "recovery_enabled"; public static final String SETTING_NAME_SKIP_FAILURE = "skip_failure"; - - public static final String SETTING_NAME_NAME = "name"; - - public static final String SETTING_NAME_REPOSITORY_SETTINGS = "repository_settings"; + static final String SETTING_NAME_NAME = "name"; + static final String SETTING_NAME_REPOSITORY_SETTINGS = "repository_settings"; /** * When specified under the "service_settings" section, it indicates whether credential store @@ -64,15 +63,14 @@ public class Setting { * } * */ - public static final String SETTING_NAME_CREDENTIAL_STORE_ENABLED = "credential_store_enabled"; + private static final String SETTING_NAME_CREDENTIAL_STORE_ENABLED = "credential_store_enabled"; /** * Settings. * * @param properties setting name-->Set(property name-->property value) */ - public Setting(Map>> properties) { - + public Setting(Map>> properties) { this.properties = properties; } @@ -81,7 +79,7 @@ public Setting(Map>> properties) { * * @return map of properties for this settings instance keyed by setting name. */ - public Map>> getProperties() { + public Map>> getProperties() { return properties; } @@ -91,11 +89,101 @@ public Map>> getProperties() { * @param settingName * @return Set of Map of properties. */ - public Set> getSettingValue(String settingName) { - if (properties.containsKey(settingName)) { - return properties.get(settingName); + public Set> getSettingValue(String settingName) { + return properties.getOrDefault(settingName, ImmutableSet.of()); + } + + /** + * Get whether the specified service is enabled for credential store use. + * + *
+   *     {@code
+   *       {
+   *         "service_settings" : [
+   *         { "name" : "RANGER",
+   *           "recovery_enabled" : "true",
+   *           "credential_store_enabled" : "true"
+   *         },
+   *         { "name" : "HIVE",
+   *           "recovery_enabled" : "true",
+   *           "credential_store_enabled" : "false"
+   *         },
+   *         { "name" : "TEZ",
+   *           "recovery_enabled" : "false"
+   *         }
+   *       ]
+   *     }
+   *   }
+   * 
+ * + * @param serviceName - Service name. + * + * @return null if value is not specified; true or false if specified. + */ + String getCredentialStoreEnabled(String serviceName) { + return getStringFromNamedMap(SETTING_NAME_SERVICE_SETTINGS, serviceName, SETTING_NAME_CREDENTIAL_STORE_ENABLED); + } + + private String getStringFromNamedMap(String outerKey, String mapName, String innerKey) { + Set> maps = getSettingValue(outerKey); + for (Map map : maps) { + String name = map.get(SETTING_NAME_NAME); + if (Objects.equals(name, mapName)) { + String value = map.get(innerKey); + if (!StringUtils.isEmpty(value)) { + return value; + } + break; + } + } + return null; + } + + private String getString(String outerKey, String innerKey) { + Set> maps = getSettingValue(outerKey); + for (Map map : maps) { + if (map.containsKey(innerKey)) { + return map.get(innerKey); + } + } + return null; + } + + public boolean shouldSkipFailure() { + return Boolean.parseBoolean(getString(SETTING_NAME_DEPLOYMENT_SETTINGS, SETTING_NAME_SKIP_FAILURE)); + } + + List processRepoSettings() { + Set> repositorySettingsValue = getSettingValue(SETTING_NAME_REPOSITORY_SETTINGS); + return repositorySettingsValue.stream() + .map(RepositorySetting::fromMap) + .collect(toList()); + } + + /** + * Get whether the specified component in the service is enabled + * for auto start. + * + * @param serviceName - Service name. + * @param componentName - Component name. + * + * @return null if value is not specified; true or false if specified. + */ + String getRecoveryEnabled(String serviceName, String componentName) { + Set> maps; + + // If component name was specified in the list of "component_settings", + // determine if recovery_enabled is true or false and return it. + String result = getStringFromNamedMap(SETTING_NAME_COMPONENT_SETTINGS, componentName, SETTING_NAME_RECOVERY_ENABLED); + // If component name is not specified, look up it's service. + if (result == null) { + result = getStringFromNamedMap(SETTING_NAME_SERVICE_SETTINGS, serviceName, SETTING_NAME_RECOVERY_ENABLED); + } + // If service name is not specified, look up the cluster setting. + if (result == null) { + result = getString(SETTING_NAME_RECOVERY_SETTINGS, SETTING_NAME_RECOVERY_ENABLED); } - return Collections.emptySet(); + return result; } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java index a01f198ab1a..ff002821179 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/SettingFactory.java @@ -64,23 +64,20 @@ public class SettingFactory { /** * Attempts to build the list of settings in the following format: * setting_name1-->[propertyName1-->propertyValue1, propertyName2-->propertyValue2] - * @param blueprintSetting - * @return */ public static Setting getSetting(Collection> blueprintSetting) { - Map>> properties = new HashMap<>(); - Setting setting = new Setting(properties); + Map>> properties = new HashMap<>(); if (blueprintSetting != null) { for (Map settingMap : blueprintSetting) { for (Map.Entry entry : settingMap.entrySet()) { final String[] propertyNames = entry.getKey().split("/"); - Set> settingValue; + Set> settingValue; if (entry.getValue() instanceof Set) { - settingValue = (HashSet>)entry.getValue(); + settingValue = (Set>) entry.getValue(); } else if (propertyNames.length > 1){ - HashMap property = new HashMap<>(); + Map property = new HashMap<>(); property.put(propertyNames[1], String.valueOf(entry.getValue())); settingValue = properties.get(propertyNames[0]); if (settingValue == null) { @@ -96,6 +93,6 @@ else if (propertyNames.length > 1){ } } - return setting; + return new Setting(properties); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java new file mode 100644 index 00000000000..9d524d027c0 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackComponentResolver.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static java.util.stream.Collectors.toSet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.state.StackId; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Joiner; +import com.google.common.base.Strings; + +public class StackComponentResolver implements ComponentResolver { + + private static final Logger LOG = LoggerFactory.getLogger(StackComponentResolver.class); + + @Override + public Map> resolveComponents(BlueprintBasedClusterProvisionRequest request) { + Map uniqueServices = request.getUniqueServices(); + Map> mpackServices = request.getServicesByMpack(); + + Map> result = new HashMap<>(); + List problems = new LinkedList<>(); + + StackDefinition stack = request.getStack(); + for (HostGroup hg : request.getHostGroups().values()) { + result.put(hg.getName(), new HashSet<>()); + + for (Component comp : hg.getComponents()) { + Stream> servicesForComponent = stack.getServicesForComponent(comp.getName()); + servicesForComponent = filterByMpackName(comp, servicesForComponent); + servicesForComponent = filterByServiceName(comp, servicesForComponent, mpackServices, uniqueServices); + + Set> serviceMatches = servicesForComponent.collect(toSet()); + + if (serviceMatches.size() != 1) { + String msg = formatResolutionProblemMessage(hg, comp, serviceMatches); + LOG.warn("Component resolution failure:" + msg); + problems.add(msg); + } else { + Pair stackService = serviceMatches.iterator().next(); + StackId stackId = stackService.getLeft(); + String serviceType = stackService.getRight(); + + ResolvedComponent resolved = ResolvedComponent.builder(comp) + .stackId(stackId) + .serviceType(serviceType) + .build(); + + LOG.debug("Component resolved: " + resolved); + result.get(hg.getName()).add(resolved); + } + } + } + + if (!problems.isEmpty()) { + throw new IllegalArgumentException("Component resolution failure:\n" + Joiner.on("\n").join(problems)); + } + + return result; + } + + private static String formatResolutionProblemMessage(HostGroup hg, Component comp, Set> serviceMatches) { + boolean multipleMatches = !serviceMatches.isEmpty(); + String problem = multipleMatches ? "Multiple services" : "No service"; + + StringBuilder sb = new StringBuilder(problem) + .append(" found for component ").append(comp.getName()) + .append(" in host group " ).append(hg.getName()); + + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + sb.append(" mpack: ").append(comp.getMpackInstance()); + } + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + sb.append(" service: ").append(comp.getServiceInstance()); + } + if (multipleMatches) { + sb.append(": ").append(serviceMatches); + } + + return sb.toString(); + } + + // if component references a specific service instance, filter the stream by the type of that service + private static Stream> filterByServiceName(Component comp, Stream> stream, + Map> mpackServices, Map uniqueServices + ) { + if (!Strings.isNullOrEmpty(comp.getServiceInstance())) { + String mpackName = comp.getMpackInstance(); + Map services = !Strings.isNullOrEmpty(mpackName) + ? mpackServices.get(mpackName) + : uniqueServices; + + ServiceInstance service = services.get(comp.getServiceInstance()); + if (service != null) { + String serviceType = service.getType(); + + return stream.filter(pair -> pair.getRight().equals(serviceType)); + } + } + + return stream; + } + + // if component references a specific mpack instance, filter the stream by the name of that mpack + private static Stream> filterByMpackName(Component comp, Stream> stream) { + if (!Strings.isNullOrEmpty(comp.getMpackInstance())) { + return stream.filter(pair -> pair.getLeft().getStackName().equals(comp.getMpackInstance())); + } + + return stream; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java new file mode 100644 index 00000000000..d7b07401603 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/StackFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import org.apache.ambari.server.StackAccessException; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.StackId; + +/** + * Internal interface used to abstract out the process of creating the Stack object. + * + * This is used to simplify unit testing, since a new Factory can be provided to + * simulate various Stack or error conditions. + */ +public interface StackFactory { + Stack createStack(StackId stackId) throws StackAccessException; +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 51eedd31e23..81082aacf1c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -51,7 +51,6 @@ import org.apache.ambari.server.controller.internal.ProvisionClusterRequest; import org.apache.ambari.server.controller.internal.RequestImpl; import org.apache.ambari.server.controller.internal.ScaleClusterRequest; -import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; @@ -130,6 +129,9 @@ public class TopologyManager { //todo: currently only support a single cluster private Map clusterTopologyMap = new HashMap<>(); + @Inject + private Configuration configuration; + @Inject private StackAdvisorBlueprintProcessor stackAdvisorBlueprintProcessor; @@ -156,6 +158,9 @@ public class TopologyManager { @Inject private TopologyValidatorService topologyValidatorService; + @Inject + private ComponentResolver resolver; + /** * A boolean not cached thread-local (volatile) to prevent double-checked * locking on the synchronized keyword. @@ -206,7 +211,7 @@ private void ensureInitialized() { // ensure KERBEROS_CLIENT is present in each hostgroup even if it's not in original BP for(ClusterTopology clusterTopology : clusterTopologyMap.values()) { if (clusterTopology.isClusterKerberosEnabled()) { - addKerberosClient(clusterTopology); + clusterTopology.getBlueprint().ensureKerberosClientIsPresent(); } } isInitialized = true; @@ -234,12 +239,12 @@ public void onRequestFinished(RequestFinishedEvent event) { if(isLogicalRequestSuccessful(provisionRequest)) { LOG.info("Cluster creation request id={} using Blueprint {} successfully completed for cluster id={}", clusterProvisionWithBlueprintCreateRequests.get(event.getClusterId()).getRequestId(), - clusterTopologyMap.get(event.getClusterId()).getBlueprint().getName(), + clusterTopologyMap.get(event.getClusterId()).getBlueprintName(), event.getClusterId()); } else { LOG.info("Cluster creation request id={} using Blueprint {} failed for cluster id={}", clusterProvisionWithBlueprintCreateRequests.get(event.getClusterId()).getRequestId(), - clusterTopologyMap.get(event.getClusterId()).getBlueprint().getName(), + clusterTopologyMap.get(event.getClusterId()).getBlueprintName(), event.getClusterId()); } } @@ -275,55 +280,35 @@ public boolean isClusterProvisionWithBlueprintFinished(long clusterId) { public RequestStatusResponse provisionCluster(final ProvisionClusterRequest request) throws InvalidTopologyException, AmbariException { ensureInitialized(); - final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request); + BlueprintBasedClusterProvisionRequest provisionRequest = new BlueprintBasedClusterProvisionRequest(ambariContext, securityConfigurationFactory, request.getBlueprint(), request); + Map> resolved = resolver.resolveComponents(provisionRequest); + + final ClusterTopologyImpl topology = new ClusterTopologyImpl(ambariContext, provisionRequest, resolved); final String clusterName = request.getClusterName(); - final StackDefinition stack = topology.getBlueprint().getStack(); final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); + final SecurityConfiguration securityConfiguration = provisionRequest.getSecurity(); + + topologyValidatorService.validateTopologyConfiguration(topology); // FIXME known stacks validation is too late here // get the id prior to creating ambari resources which increments the counter final Long provisionId = ambariContext.getNextRequestId(); - SecurityType securityType = null; - Credential credential = null; - - SecurityConfiguration securityConfiguration = processSecurityConfiguration(request); - - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS) { - securityType = SecurityType.KERBEROS; - addKerberosClient(topology); - - // refresh default stack config after adding KERBEROS_CLIENT component to topology - topology.getBlueprint().getConfiguration().setParentConfiguration(stack.getConfiguration(topology.getBlueprint().getServices())); - - credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); - if (credential == null) { - throw new InvalidTopologyException(KDC_ADMIN_CREDENTIAL + " is missing from request."); - } - } - - topologyValidatorService.validateTopologyConfiguration(topology); - - // create resources - ambariContext.createAmbariResources(topology, clusterName, securityType, repoVersion, repoVersionID); + ambariContext.createAmbariResources(topology, clusterName, securityConfiguration.getType(), repoVersion, repoVersionID); - if (securityConfiguration != null && securityConfiguration.getDescriptor() != null) { + if (securityConfiguration.getDescriptor() != null) { submitKerberosDescriptorAsArtifact(clusterName, securityConfiguration.getDescriptor()); } - if (credential != null) { + if (securityConfiguration.getType() == SecurityType.KERBEROS) { + Credential credential = request.getCredentialsMap().get(KDC_ADMIN_CREDENTIAL); submitCredential(clusterName, credential); } long clusterId = ambariContext.getClusterId(clusterName); topology.setClusterId(clusterId); request.setClusterId(clusterId); - // set recommendation strategy - topology.setConfigRecommendationStrategy(request.getConfigRecommendationStrategy()); - // set provision action requested - topology.setProvisionAction(request.getProvisionAction()); - // create task executor for TopologyTasks getOrCreateTopologyTaskExecutor(clusterId); @@ -341,7 +326,7 @@ public LogicalRequest call() throws Exception { clusterTopologyMap.put(clusterId, topology); addClusterConfigRequest(logicalRequest, topology, new ClusterConfigurationRequest(ambariContext, topology, true, - stackAdvisorBlueprintProcessor, securityType == SecurityType.KERBEROS)); + stackAdvisorBlueprintProcessor, securityConfiguration.getType() == SecurityType.KERBEROS)); // Process the logical request processRequest(request, topology, logicalRequest); @@ -349,7 +334,7 @@ public LogicalRequest call() throws Exception { //todo: this should be invoked as part of a generic lifecycle event which could possibly //todo: be tied to cluster state - StackId stackId = Iterables.getFirst(topology.getBlueprint().getStackIds(), null); // FIXME need for stackId in ClusterRequest will be removed + StackId stackId = Iterables.getFirst(topology.getStackIds(), null); // FIXME need for stackId in ClusterRequest will be removed ambariContext.persistInstallStateForUI(clusterName, stackId); clusterProvisionWithBlueprintCreateRequests.put(clusterId, logicalRequest); return getRequestStatus(logicalRequest.getRequestId()); @@ -421,33 +406,10 @@ private void submitCredential(String clusterName, Credential credential) { } - /** - * Retrieve security info from Blueprint if missing from Cluster Template request. - * - * @param request - * @return - */ - private SecurityConfiguration processSecurityConfiguration(ProvisionClusterRequest request) { - LOG.debug("Getting security configuration from the request ..."); - SecurityConfiguration securityConfiguration = request.getSecurityConfiguration(); - - if (securityConfiguration == null) { - // todo - perform this logic at request creation instead! - LOG.debug("There's no security configuration in the request, retrieving it from the associated blueprint"); - securityConfiguration = request.getBlueprint().getSecurity(); - if (securityConfiguration != null && securityConfiguration.getType() == SecurityType.KERBEROS && - securityConfiguration.getDescriptorReference() != null) { - securityConfiguration = securityConfigurationFactory.loadSecurityConfigurationByReference - (securityConfiguration.getDescriptorReference()); - } - } - return securityConfiguration; - } - private void submitKerberosDescriptorAsArtifact(String clusterName, String descriptor) { ResourceProvider artifactProvider = - ambariContext.getClusterController().ensureResourceProvider(Resource.Type.Artifact); + AmbariContext.getClusterController().ensureResourceProvider(Resource.Type.Artifact); Map properties = new HashMap<>(); properties.put(ArtifactResourceProvider.ARTIFACT_NAME_PROPERTY, "kerberos_descriptor"); @@ -851,8 +813,6 @@ private void processRequest(TopologyRequest request, ClusterTopology topology, f LOG.info("TopologyManager.processRequest: Entering"); - finalizeTopology(request, topology); - boolean requestHostComplete = false; //todo: overall synchronization. Currently we have nested synchronization here @@ -1083,25 +1043,10 @@ private boolean isLogicalRequestSuccessful(LogicalRequest logicalRequest) { return logicalRequest != null && logicalRequest.isSuccessful(); } - //todo: this should invoke a callback on each 'service' in the topology - private void finalizeTopology(TopologyRequest request, ClusterTopology topology) { - } - private boolean isHostIgnored(String host) { return hostsToIgnore.remove(host); } - /** - * Add the kerberos client to groups if kerberos is enabled for the cluster. - * - * @param topology cluster topology - */ - private void addKerberosClient(ClusterTopology topology) { - for (HostGroup group : topology.getBlueprint().getHostGroups().values()) { - group.addComponent(new Component("KERBEROS_CLIENT")); - } - } - /** * Register the configuration task which is responsible for configuration topology resolution * and setting the updated configuration on the cluster. This task needs to be submitted to the diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java index bd5630b9875..94fcf23e3b1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequest.java @@ -43,10 +43,6 @@ enum Type { PROVISION, SCALE, EXPORT } */ Type getType(); - //todo: only a single BP may be specified so all host groups have the same bp. - //todo: BP really needs to be associated with the HostGroupInfo, even for create which will have a single BP - //todo: for all HG's. - /** * Get the blueprint instance associated with the request. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java index ff79f526ea6..44f0d1f02ac 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyRequestFactoryImpl.java @@ -27,6 +27,7 @@ * Factory for creating topology requests. */ public class TopologyRequestFactoryImpl implements TopologyRequestFactory { + @Override public ProvisionClusterRequest createProvisionClusterRequest(Map properties, SecurityConfiguration securityConfiguration) throws InvalidTopologyTemplateException { return new ProvisionClusterRequest(properties, securityConfiguration); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java index 74fb9b1c897..4ae136b6dba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java @@ -17,13 +17,7 @@ */ package org.apache.ambari.server.topology.tasks; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,16 +41,10 @@ public Type getType() { @Override public void runTask() { - LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostRequest.getHostName()); - - HostGroup group = hostRequest.getHostGroup(); - Map> serviceComponents = new HashMap<>(); - for (String service : group.getServices()) { - serviceComponents.put(service, new HashSet<>(group.getComponentNames(service))); - } - clusterTopology.getAmbariContext().createAmbariHostResources(hostRequest.getClusterId(), - hostRequest.getHostName(), serviceComponents); - - LOG.info("HostRequest: Exiting RESOURCE_CREATION task for host: {}", hostRequest.getHostName()); + String hostName = hostRequest.getHostName(); + LOG.info("HostRequest: Executing RESOURCE_CREATION task for host: {}", hostName); + long clusterId = hostRequest.getClusterId(); + clusterTopology.getAmbariContext().createAmbariHostResources(clusterId, hostName, clusterTopology.getComponentsInHostGroup(hostRequest.getHostgroupName())); + LOG.info("HostRequest: Exiting RESOURCE_CREATION task for host: {}", hostName); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java new file mode 100644 index 00000000000..82ea759f457 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BasicBlueprintValidator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.Blueprint; + +public class BasicBlueprintValidator implements BlueprintValidator { + + @Override + public void validate(Blueprint blueprint) throws IllegalArgumentException { + // FIXME implement + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java similarity index 51% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java index 64e08ad1b5c..53d58785a8b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/BlueprintValidator.java @@ -16,27 +16,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.Blueprint; /** - * Provides blueprint validation. + * Provides basic blueprint validation. */ public interface BlueprintValidator { - /** - * Validate blueprint topology. - * - * @param blueprint the blueprint to validate - * @throws InvalidTopologyException if the topology is invalid - */ - void validateTopology(Blueprint blueprint) throws InvalidTopologyException; - /** - * Validate that required properties are provided. - * This doesn't include password properties. - * - * @param blueprint the blueprint to validate - * @throws InvalidTopologyException if required properties are not set in blueprint - * @throws GPLLicenseNotAcceptedException if the blueprint requires use of GPL software, but GPL license was not accepted - */ - void validateRequiredProperties(Blueprint blueprint) throws InvalidTopologyException, GPLLicenseNotAcceptedException; + void validate(Blueprint blueprint) throws IllegalArgumentException; + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java index 8bcbcffbdc5..0a10039b6a1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ChainedTopologyValidator.java @@ -32,19 +32,18 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Topology validator wrapper implementation. Executes a set of validations by calling a preconfgured set of validator implementations. + * Topology validator wrapper implementation. Executes a set of validations by calling a preconfigured set of validator implementations. */ public class ChainedTopologyValidator implements TopologyValidator { private static final Logger LOGGER = LoggerFactory.getLogger(ChainedTopologyValidator.class); - private List validators; + private final List validators; - public ChainedTopologyValidator(List validators) { + ChainedTopologyValidator(List validators) { this.validators = validators; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java index 0170186d390..25748695943 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidator.java @@ -18,7 +18,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +33,7 @@ public class ClusterConfigTypeValidator implements TopologyValidator { public void validate(ClusterTopology topology) throws InvalidTopologyException { // config types in from the request / configuration is always set in the request instance - Set topologyClusterConfigTypes = new HashSet(topology.getConfiguration().getAllConfigTypes()); + Set topologyClusterConfigTypes = new HashSet<>(topology.getConfiguration().getAllConfigTypes()); LOGGER.debug("Cluster config types: {}", topologyClusterConfigTypes); if (topologyClusterConfigTypes.isEmpty()) { @@ -44,8 +43,8 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { // collecting all config types for services in the blueprint (from the related stack) Set stackServiceConfigTypes = new HashSet<>(); - for (String serviceName : topology.getBlueprint().getServices()) { - stackServiceConfigTypes.addAll(topology.getBlueprint().getStack().getConfigurationTypes(serviceName)); + for (String serviceName : topology.getServices()) { + stackServiceConfigTypes.addAll(topology.getStack().getConfigurationTypes(serviceName)); } // identifying invalid config types diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java similarity index 58% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java index 39cbbfaf360..a25dcc4ce45 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidator.java @@ -16,62 +16,52 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; -import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; -import java.util.Set; -import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.AutoDeployInfo; import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.utils.SecretReference; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.inject.Inject; - /** - * Default blueprint validator. + * Verifies that service dependencies and component cardinality requirements are satisfied. */ -public class BlueprintValidatorImpl implements BlueprintValidator { - - private static final Logger LOGGER = LoggerFactory.getLogger(BlueprintValidatorImpl.class); - - public static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class"; - public static final String CODEC_CLASSES_PROPERTY_NAME = "io.compression.codecs"; - public static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec"; +public class DependencyAndCardinalityValidator implements TopologyValidator { - private final Configuration configuration; - - @Inject - public BlueprintValidatorImpl(Configuration configuration) { - this.configuration = configuration; - } + private static final Logger LOGGER = LoggerFactory.getLogger(DependencyAndCardinalityValidator.class); @Override - public void validateTopology(Blueprint blueprint) throws InvalidTopologyException { - LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName()); + public void validate(ClusterTopology topology) throws InvalidTopologyException { + Blueprint blueprint = topology.getBlueprint(); + LOGGER.info("Validating topology for blueprint: [{}]", topology.getBlueprintName()); - StackDefinition stack = blueprint.getStack(); + StackDefinition stack = topology.getStack(); Collection hostGroups = blueprint.getHostGroups().values(); Map>> missingDependencies = new HashMap<>(); for (HostGroup group : hostGroups) { - Map> missingGroupDependencies = validateHostGroup(blueprint, stack, group); + Map> missingGroupDependencies = validateHostGroup(topology, blueprint, stack, group); if (!missingGroupDependencies.isEmpty()) { missingDependencies.put(group.getName(), missingGroupDependencies); } } Collection cardinalityFailures = new HashSet<>(); - Collection services = blueprint.getServices(); + Collection services = topology.getServices(); for (String service : services) { for (String component : stack.getComponents(service)) { @@ -81,7 +71,7 @@ public void validateTopology(Blueprint blueprint) throws InvalidTopologyExceptio cardinalityFailures.addAll(verifyComponentInAllHostGroups(blueprint, new Component(component), autoDeploy)); } else { cardinalityFailures.addAll(verifyComponentCardinalityCount( - stack, blueprint, new Component(component), cardinality, autoDeploy)); + stack, topology, blueprint, new Component(component), cardinality, autoDeploy)); } } } @@ -91,94 +81,6 @@ public void validateTopology(Blueprint blueprint) throws InvalidTopologyExceptio } } - @Override - public void validateRequiredProperties(Blueprint blueprint) throws InvalidTopologyException, GPLLicenseNotAcceptedException { - - // we don't want to include default stack properties so we can't just use hostGroup full properties - Map> clusterConfigurations = blueprint.getConfiguration().getProperties(); - - // we need to have real passwords, not references - if (clusterConfigurations != null) { - - // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it - boolean gplEnabled = configuration.getGplLicenseAccepted(); - - StringBuilder errorMessage = new StringBuilder(); - boolean containsSecretReferences = false; - for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { - String configType = configEntry.getKey(); - if (configEntry.getValue() != null) { - for (Map.Entry propertyEntry : configEntry.getValue().entrySet()) { - String propertyName = propertyEntry.getKey(); - String propertyValue = propertyEntry.getValue(); - if (propertyValue != null) { - if (!gplEnabled && configType.equals("core-site") - && (propertyName.equals(LZO_CODEC_CLASS_PROPERTY_NAME) || propertyName.equals(CODEC_CLASSES_PROPERTY_NAME)) - && propertyValue.contains(LZO_CODEC_CLASS)) { - throw new GPLLicenseNotAcceptedException("Your Ambari server has not been configured to download LZO GPL software. " + - "Please refer to documentation to configure Ambari before proceeding."); - } - if (SecretReference.isSecret(propertyValue)) { - errorMessage.append(" Config:" + configType + " Property:" + propertyName + "\n"); - containsSecretReferences = true; - } - } - } - } - } - if (containsSecretReferences) { - throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + - "replace following properties with real passwords:\n" + errorMessage); - } - } - - - for (HostGroup hostGroup : blueprint.getHostGroups().values()) { - Collection processedServices = new HashSet<>(); - Map> allRequiredProperties = new HashMap<>(); - Map> operationalConfiguration = new HashMap<>(clusterConfigurations); - - operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); - for (String component : hostGroup.getComponentNames()) { - //check that MYSQL_SERVER component is not available while hive is using existing db - if (component.equals("MYSQL_SERVER")) { - Map hiveEnvConfig = clusterConfigurations.get("hive-env"); - if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null - && hiveEnvConfig.get("hive_database").startsWith("Existing")) { - throw new InvalidTopologyException("Incorrect configuration: MYSQL_SERVER component is available but hive" + - " using existing db!"); - } - } - if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE")) { - Map hadoopEnvConfig = clusterConfigurations.get("hadoop-env"); - if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { - ArrayList hostGroupsForComponent = new ArrayList<>(blueprint.getHostGroupsForComponent(component)); - Set givenHostGroups = new HashSet<>(); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); - givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); - if(givenHostGroups.size() != hostGroupsForComponent.size()) { - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - if(HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && HostGroup.HOSTGROUP_REGEX.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()){ - for (HostGroup hostGroupForComponent : hostGroupsForComponent) { - Iterator itr = givenHostGroups.iterator(); - while(itr.hasNext()){ - if(itr.next().contains(hostGroupForComponent.getName())){ - itr.remove(); - } - } - } - } - - if(!givenHostGroups.isEmpty()){ - throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); - } - } - } - } - } - } - /** * Verify that a component is included in all host groups. * For components that are auto-install enabled, will add component to topology if needed. @@ -206,7 +108,7 @@ private Collection verifyComponentInAllHostGroups(Blueprint blueprint, C return cardinalityFailures; } - private Map> validateHostGroup(Blueprint blueprint, StackDefinition stack, HostGroup group) { + private Map> validateHostGroup(ClusterTopology topology, Blueprint blueprint, StackDefinition stack, HostGroup group) { LOGGER.info("Validating hostgroup: {}", group.getName()); Map> missingDependencies = new HashMap<>(); @@ -216,18 +118,11 @@ private Map> validateHostGroup(Blueprint blue for (DependencyInfo dependency : stack.getDependenciesForComponent(component.getName())) { LOGGER.debug("Processing dependency [{}] for component [{}]", dependency.getName(), component); - String conditionalService = stack.getConditionalServiceForDependency(dependency); - if (conditionalService != null && !blueprint.getServices().contains(conditionalService)) { - LOGGER.debug("Conditional service [{}] is missing from the blueprint, skipping dependency [{}]", - conditionalService, dependency.getName()); - continue; - } - // dependent components from the stack definitions are only added if related services are explicitly added to the blueprint! boolean isClientDependency = stack.getComponentInfo(dependency.getComponentName()).isClient(); - if (isClientDependency && !blueprint.getServices().contains(dependency.getServiceName())) { + if (isClientDependency && !topology.getServices().contains(dependency.getServiceName())) { LOGGER.debug("The service [{}] for component [{}] is missing from the blueprint [{}], skipping dependency", - dependency.getServiceName(), dependency.getComponentName(), blueprint.getName()); + dependency.getServiceName(), dependency.getComponentName(), topology.getBlueprintName()); continue; } @@ -240,7 +135,7 @@ private Map> validateHostGroup(Blueprint blue if(dependency.hasDependencyConditions()) { boolean conditionsSatisfied = true; for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions()) { - if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties())) { + if (!dependencyCondition.isResolved(topology.getConfiguration().getFullProperties())) { conditionsSatisfied = false; break; } @@ -251,7 +146,7 @@ private Map> validateHostGroup(Blueprint blue } if (dependencyScope.equals("cluster")) { Collection missingDependencyInfo = verifyComponentCardinalityCount( - stack, blueprint, new Component(componentName), new Cardinality("1+"), autoDeployInfo); + stack, topology, blueprint, new Component(componentName), new Cardinality("1+"), autoDeployInfo); resolved = missingDependencyInfo.isEmpty(); } else if (dependencyScope.equals("host")) { @@ -289,15 +184,16 @@ private Map> validateHostGroup(Blueprint blue */ private Collection verifyComponentCardinalityCount( StackDefinition stack, + ClusterTopology topology, Blueprint blueprint, Component component, Cardinality cardinality, AutoDeployInfo autoDeploy ) { - Map> configProperties = blueprint.getConfiguration().getProperties(); + Map> configProperties = topology.getConfiguration().getProperties(); Collection cardinalityFailures = new HashSet<>(); //todo: don't hard code this HA logic here - if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && + if (BlueprintConfigurationProcessor.isNameNodeHAEnabled(configProperties) && (component.getName().equals("SECONDARY_NAMENODE"))) { // override the cardinality for this component in an HA deployment, // since the SECONDARY_NAMENODE should not be started in this scenario diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java new file mode 100644 index 00000000000..5ce4940fe50 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/GplPropertiesValidator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableSet; + +/** + * Properties that indicate usage of GPL software are + * allowed with explicit approval from user. + */ +public class GplPropertiesValidator implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(GplPropertiesValidator.class); + + private static final String CORE_SITE = "core-site"; + static final String LZO_CODEC_CLASS_PROPERTY_NAME = "io.compression.codec.lzo.class"; + static final String CODEC_CLASSES_PROPERTY_NAME = "io.compression.codecs"; + static final String LZO_CODEC_CLASS = "com.hadoop.compression.lzo.LzoCodec"; + + private static final Set PROPERTY_NAMES = ImmutableSet.of( + LZO_CODEC_CLASS_PROPERTY_NAME, + CODEC_CLASSES_PROPERTY_NAME + ); + + private static final String GPL_LICENSE_ERROR_MESSAGE = + "Your Ambari server has not been configured to download LZO GPL software. " + + "Please refer to documentation to configure Ambari before proceeding."; + + private final Configuration configuration; + + @Inject + public GplPropertiesValidator(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + // need to reject blueprints that have LZO enabled if the Ambari Server hasn't been configured for it + boolean gplEnabled = configuration.getGplLicenseAccepted(); + + if (gplEnabled) { + LOG.info("GPL license accepted, skipping config check"); + return; + } + + // we don't want to include default stack properties so we can't use full properties + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + if (clusterConfigurations != null) { + Map properties = clusterConfigurations.get(CORE_SITE); + if (properties != null) { + for (String propertyName : PROPERTY_NAMES) { + String propertyValue = properties.get(propertyName); + if (propertyValue != null && propertyValue.contains(LZO_CODEC_CLASS)) { + throw new GPLLicenseNotAcceptedException(GPL_LICENSE_ERROR_MESSAGE); + } + } + } + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index 9d9ad00a35b..6ba9a2faf8f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -17,7 +17,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,14 +30,13 @@ public class HiveServiceValidator implements TopologyValidator { private static final String HIVE_DB_DEFAULT = "New MySQL Database"; private static final String HIVE_DB_PROPERTY = "hive_database"; private static final String MYSQL_SERVER_COMPONENT = "MYSQL_SERVER"; - public static final String HIVE_SERVICE = "HIVE"; + private static final String HIVE_SERVICE = "HIVE"; @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { - // there is no hive configured in the blueprint, nothing to do (does the validator apply?) - if (!topology.getBlueprint().getServices().contains(HIVE_SERVICE)) { + if (!topology.getServices().contains(HIVE_SERVICE)) { LOGGER.info(" [{}] service is not listed in the blueprint, skipping hive service validation.", HIVE_SERVICE); return; } @@ -52,16 +50,22 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { throw new InvalidTopologyException(errorMessage); } - // hive database has custom configuration, skipping validation - if (!HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY))) { - LOGGER.info("Custom hive database settings detected. HIVE service validation succeeded."); - return; + boolean hiveWantsMysql = HIVE_DB_DEFAULT.equals(clusterConfiguration.getPropertyValue(HIVE_ENV, HIVE_DB_PROPERTY)); + boolean topologyContainsMysql = topology.getComponents() + .anyMatch(c -> MYSQL_SERVER_COMPONENT.equals(c.componentName()) && HIVE_SERVICE.equals(c.effectiveServiceName())); + + if (topologyContainsMysql && !hiveWantsMysql) { + String errorMessage = String.format( + "Incorrect configuration: %s component is specified in blueprint, but Hive is configured to use existing DB", + MYSQL_SERVER_COMPONENT); + LOGGER.error(errorMessage); + throw new InvalidTopologyException(errorMessage); } - // hive database settings need the mysql-server component in the blueprint - if (!topology.getBlueprint().getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) { - String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " + - "is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT); + if (hiveWantsMysql && !topologyContainsMysql) { + String errorMessage = String.format( + "The component %s must explicitly be specified in the blueprint if Hive database is configured with %s.", + MYSQL_SERVER_COMPONENT, HIVE_DB_DEFAULT); LOGGER.error(errorMessage); throw new InvalidTopologyException(errorMessage); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java new file mode 100644 index 00000000000..fbfb03da890 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/NameNodeHighAvailabilityValidator.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology.validators; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.controller.internal.BlueprintConfigurationProcessor; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Verifies that dfs_ha_initial_namenode_active and dfs_ha_initial_namenode_standby properties + * in hadoop-env reference host groups with NAMENODE components. + */ +public class NameNodeHighAvailabilityValidator implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(NameNodeHighAvailabilityValidator.class); + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + Blueprint blueprint = topology.getBlueprint(); + + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + if (!BlueprintConfigurationProcessor.isNameNodeHAEnabled(clusterConfigurations)) { + LOG.info("NAMENODE HA is not enabled, skipping validation of {}", blueprint.getName()); + return; + } + + LOG.info("Validating NAMENODE HA for blueprint: {}", blueprint.getName()); + + List hostGroupsForComponent = new ArrayList<>(topology.getHostGroupsForComponent("NAMENODE")); + + for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { + Map> operationalConfiguration = new HashMap<>(clusterConfigurations); + + operationalConfiguration.putAll(hostGroup.getConfiguration().getProperties()); + if (hostGroup.getComponentNames().contains("NAMENODE")) { + Map hadoopEnvConfig = operationalConfiguration.get("hadoop-env"); + if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active") && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) { + Set givenHostGroups = new HashSet<>(); + givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")); + givenHostGroups.add(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")); + if(givenHostGroups.size() != hostGroupsForComponent.size()) { + throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); + } + if (BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_active")).matches() && BlueprintConfigurationProcessor.HOST_GROUP_PLACEHOLDER_PATTERN.matcher(hadoopEnvConfig.get("dfs_ha_initial_namenode_standby")).matches()) { + for (String hostGroupForComponent : hostGroupsForComponent) { + givenHostGroups.removeIf(s -> s.contains(hostGroupForComponent)); + } + } + + if(!givenHostGroups.isEmpty()){ + throw new IllegalArgumentException("NAMENODE HA host groups mapped incorrectly for properties 'dfs_ha_initial_namenode_active' and 'dfs_ha_initial_namenode_standby'. Expected Host groups are :" + hostGroupsForComponent); + } + } + } + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java new file mode 100644 index 00000000000..e41fe2708ff --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownComponents.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.joining; + +import org.apache.ambari.server.controller.RootComponent; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RejectUnknownComponents implements TopologyValidator { + + private static final Logger LOG = LoggerFactory.getLogger(RejectUnknownComponents.class); + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + String unknownComponents = topology.getComponents() + .map(ResolvedComponent::componentName) + .filter(c -> !RootComponent.AMBARI_SERVER.name().equals(c)) + .filter(c -> !topology.getStack().getComponents().contains(c)) + .collect(joining(", ")); + + if (!unknownComponents.isEmpty()) { + String msg = "The following components are not valid for the specified stacks: " + unknownComponents; + LOG.info(msg); + throw new InvalidTopologyException(msg); + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java new file mode 100644 index 00000000000..9821bc8582a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RejectUnknownStacks.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.joining; + +import javax.inject.Provider; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; + +/** + * Verifies that the topology only references known stacks. + */ +public class RejectUnknownStacks implements TopologyValidator { + + private final Provider metaInfo; + + RejectUnknownStacks(Provider metaInfo) { + this.metaInfo = metaInfo; + } + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + String unknownStacks = topology.getStackIds().stream() + .filter(stackId -> !metaInfo.get().isKnownStack(stackId)) + .sorted() + .map(Object::toString) + .collect(joining(", ")); + + if (!unknownStacks.isEmpty()) { + throw new InvalidTopologyException("Unknown stacks found in cluster creation request: " + unknownStacks); + } + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java index 4022fcba1e4..5102fb97a74 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidator.java @@ -23,11 +23,9 @@ import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +42,7 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator { /** * Validates the configuration coming from the blueprint and cluster creation template and ensures that all the required properties are provided. - * It's expected, that a in hostrgroup containing components for a given service all required configuration for the given service is available. + * It's expected, that a in hostgroup containing components for a given service all required configuration for the given service is available. * * @param topology the topology instance holding the configuration for cluster provisioning * @throws InvalidTopologyException when there are missing configuration types or properties related to services in the blueprint @@ -53,45 +51,38 @@ public class RequiredConfigPropertiesValidator implements TopologyValidator { public void validate(ClusterTopology topology) throws InvalidTopologyException { // collect required properties - Map>> requiredPropertiesByService = getRequiredPropertiesByService(topology.getBlueprint()); + Map>> requiredPropertiesByService = getRequiredPropertiesByService(topology); // find missing properties in the cluster configuration Map> missingProperties = new TreeMap<>(); Map> topologyConfiguration = new HashMap<>(topology.getConfiguration().getFullProperties(1)); - for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { - LOGGER.debug("Processing hostgroup configurations for hostgroup: {}", hostGroup.getName()); + for (Map.Entry>> entry : requiredPropertiesByService.entrySet()) { + String service = entry.getKey(); + Map> requiredPropertiesByType = entry.getValue(); - // copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig - Map> operationalConfigurations = new HashMap<>(topologyConfiguration); + for (HostGroup hostGroup : topology.getBlueprint().getHostGroups().values()) { + LOGGER.debug("Processing configurations for service {} in hostgroup {}", service, hostGroup.getName()); - for (Map.Entry> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) { - if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) { - operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue()); - } else { - operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue()); - } - } - - for (String hostGroupService : hostGroup.getServices()) { + // copy of all configurations available in the topology hgConfig -> topologyConfig -> bpConfig + Map> operationalConfigurations = new HashMap<>(topologyConfiguration); - if (!requiredPropertiesByService.containsKey(hostGroupService)) { - // there are no required properties for the service - LOGGER.debug("There are no required properties found for hostgroup/service: [{}/{}]", hostGroup.getName(), hostGroupService); - continue; + for (Map.Entry> hostgroupConfigEntry : hostGroup.getConfiguration().getProperties().entrySet()) { + if (operationalConfigurations.containsKey(hostgroupConfigEntry.getKey())) { + operationalConfigurations.get(hostgroupConfigEntry.getKey()).putAll(hostgroupConfigEntry.getValue()); + } else { + operationalConfigurations.put(hostgroupConfigEntry.getKey(), hostgroupConfigEntry.getValue()); + } } - Map> requiredPropertiesByType = requiredPropertiesByService.get(hostGroupService); - for (String configType : requiredPropertiesByType.keySet()) { // We need a copy not to modify the original - Collection requiredPropertiesForType = new HashSet( - requiredPropertiesByType.get(configType)); + Collection requiredPropertiesForType = new HashSet<>(requiredPropertiesByType.get(configType)); if (!operationalConfigurations.containsKey(configType)) { // all required configuration is missing for the config type - missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); + missingProperties = addToMissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); continue; } @@ -100,7 +91,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { if (!requiredPropertiesForType.isEmpty()) { LOGGER.info("Found missing properties in hostgroup: {}, config type: {}, mising properties: {}", hostGroup.getName(), configType, requiredPropertiesForType); - missingProperties = addTomissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); + missingProperties = addToMissingProperties(missingProperties, hostGroup.getName(), requiredPropertiesForType); } } } @@ -119,18 +110,17 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { * Collects required properties for services in the blueprint. Configuration properties are returned by configuration type. * service -> configType -> properties * - * @param blueprint the blueprint from the cluster topology * @return a map with configuration types mapped to collections of required property names */ - private Map>> getRequiredPropertiesByService(Blueprint blueprint) { + private Map>> getRequiredPropertiesByService(ClusterTopology topology) { Map>> requiredPropertiesForServiceByType = new HashMap<>(); - for (String bpService : blueprint.getServices()) { + for (String bpService : topology.getServices()) { LOGGER.debug("Collecting required properties for the service: {}", bpService); - Collection requiredConfigsForService = blueprint.getStack().getRequiredConfigurationProperties(bpService); + Collection requiredConfigsForService = topology.getStack().getRequiredConfigurationProperties(bpService); Map> requiredPropertiesByConfigType = new HashMap<>(); for (Stack.ConfigProperty configProperty : requiredConfigsForService) { @@ -169,7 +159,7 @@ private Map>> getRequiredPropertiesByServ } - private Map> addTomissingProperties(Map> missingProperties, String hostGroup, Collection values) { + private Map> addToMissingProperties(Map> missingProperties, String hostGroup, Collection values) { Map> missing; if (missingProperties == null) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java index 5c9e0eb6555..b3a543032f9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidator.java @@ -1,5 +1,3 @@ -package org.apache.ambari.server.topology.validators; - /* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,33 +12,30 @@ * limitations under the License. */ -import java.util.Collection; +package org.apache.ambari.server.topology.validators; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toSet; + import java.util.HashMap; -import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.ambari.server.controller.RootComponent; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.internal.StackDefinition; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; -import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; +import org.apache.ambari.server.topology.ResolvedComponent; /** * Validates that all required passwords are provided. */ public class RequiredPasswordValidator implements TopologyValidator { - // todo remove the field as all the information is available in the topology being validated - private String defaultPassword; - - public RequiredPasswordValidator() { - } - /** * Validate that all required password properties have been set or that 'default_password' is specified. * @@ -49,8 +44,7 @@ public RequiredPasswordValidator() { */ public void validate(ClusterTopology topology) throws InvalidTopologyException { - defaultPassword = topology.getDefaultPassword(); - Map>> missingPasswords = validateRequiredPasswords(topology); + Map>> missingPasswords = validateRequiredPasswords(topology); if (! missingPasswords.isEmpty()) { throw new InvalidTopologyException("Missing required password properties. Specify a value for these " + @@ -68,94 +62,45 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { * * @throws IllegalArgumentException if blueprint contains invalid information */ + private Map>> validateRequiredPasswords(ClusterTopology topology) { + Map>> missingProperties = new HashMap<>(); - //todo: this is copied/pasted from Blueprint and is currently only used by validatePasswordProperties() - //todo: seems that we should have some common place for this code so it can be used by BP and here? - private Map>> validateRequiredPasswords(ClusterTopology topology) { - - Map>> missingProperties = - new HashMap<>(); + StackDefinition stack = topology.getStack(); + String defaultPassword = topology.getDefaultPassword(); + boolean hasDefaultPassword = defaultPassword != null && !defaultPassword.trim().isEmpty(); for (Map.Entry groupEntry: topology.getHostGroupInfo().entrySet()) { String hostGroupName = groupEntry.getKey(); Map> groupProperties = groupEntry.getValue().getConfiguration().getFullProperties(3); - Collection processedServices = new HashSet<>(); - Blueprint blueprint = topology.getBlueprint(); - StackDefinition stack = blueprint.getStack(); - - HostGroup hostGroup = blueprint.getHostGroup(hostGroupName); - for (String component : hostGroup.getComponentNames()) { - //for now, AMBARI is not recognized as a service in Stacks - if (component.equals(RootComponent.AMBARI_SERVER.name())) { - continue; - } - - String serviceName = stack.getServiceForComponent(component); - if (processedServices.add(serviceName)) { - //todo: do I need to subtract excluded configs? - Collection requiredProperties = - stack.getRequiredConfigurationProperties(serviceName, PropertyInfo.PropertyType.PASSWORD); - - for (Stack.ConfigProperty property : requiredProperties) { - String category = property.getType(); - String name = property.getName(); - if (! propertyExists(topology, groupProperties, category, name)) { - Map> missingHostGroupPropsMap = missingProperties.get(hostGroupName); - if (missingHostGroupPropsMap == null) { - missingHostGroupPropsMap = new HashMap<>(); - missingProperties.put(hostGroupName, missingHostGroupPropsMap); - } - Collection missingHostGroupTypeProps = missingHostGroupPropsMap.get(category); - if (missingHostGroupTypeProps == null) { - missingHostGroupTypeProps = new HashSet<>(); - missingHostGroupPropsMap.put(category, missingHostGroupTypeProps); - } - missingHostGroupTypeProps.add(name); + Map> missingPropertiesInHostGroup = topology.getComponentsInHostGroup(hostGroupName) + .filter(component -> !RootComponent.AMBARI_SERVER.name().equals(component.componentName())) + .map(ResolvedComponent::serviceType) + .distinct() + .flatMap(serviceType -> stack.getRequiredConfigurationProperties(serviceType, PropertyInfo.PropertyType.PASSWORD).stream()) + .filter(property -> !propertyExists(groupProperties, property.getType(), property.getName())) + .collect(groupingBy(Stack.ConfigProperty::getType, mapping(Stack.ConfigProperty::getName, toSet()))); + + if (!missingPropertiesInHostGroup.isEmpty()) { + if (hasDefaultPassword) { + for (Map.Entry> entry : missingPropertiesInHostGroup.entrySet()) { + String type = entry.getKey(); + for (String name : entry.getValue()) { + topology.getConfiguration().setProperty(type, name, defaultPassword); } } + } else { + missingProperties.put(hostGroupName, missingPropertiesInHostGroup); } } } return missingProperties; } - private boolean propertyExists(ClusterTopology topology, Map> props, String type, String property) { + private boolean propertyExists(Map> props, String type, String property) { Map typeProps = props.get(type); - return (typeProps != null && typeProps.containsKey(property)) || setDefaultPassword(topology, type, property); + return typeProps != null && typeProps.containsKey(property); } - /** - * Attempt to set the default password in cluster configuration for missing password property. - * - * @param configType configuration type - * @param property password property name - * - * @return true if password was set, otherwise false. Currently the password will always be set - * unless it is null - */ - private boolean setDefaultPassword(ClusterTopology topology, String configType, String property) { - boolean setDefaultPassword = false; - if (defaultPassword != null && ! defaultPassword.trim().isEmpty()) { - topology.getConfiguration().setProperty(configType, property, defaultPassword); - setDefaultPassword = true; - } - return setDefaultPassword; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - RequiredPasswordValidator that = (RequiredPasswordValidator) o; - - return defaultPassword == null ? that.defaultPassword == null : defaultPassword.equals(that.defaultPassword); - } - - @Override - public int hashCode() { - return defaultPassword != null ? defaultPassword.hashCode() : 0; - } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java new file mode 100644 index 00000000000..2e3c61b127e --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/SecretReferenceValidator.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import java.util.Map; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.utils.SecretReference; + +/** + * Secret references are not allowed in blueprints. + * @see SecretReference + */ +public class SecretReferenceValidator implements TopologyValidator { + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + // we don't want to include default stack properties so we can't use full properties + Map> clusterConfigurations = topology.getConfiguration().getProperties(); + + // we need to have real passwords, not references + if (clusterConfigurations != null) { + StringBuilder errorMessage = new StringBuilder(); + boolean containsSecretReferences = false; + for (Map.Entry> configEntry : clusterConfigurations.entrySet()) { + String configType = configEntry.getKey(); + Map configEntryValue = configEntry.getValue(); + if (configEntryValue != null) { + for (Map.Entry propertyEntry : configEntryValue.entrySet()) { + String propertyName = propertyEntry.getKey(); + String propertyValue = propertyEntry.getValue(); + if (propertyValue != null && SecretReference.isSecret(propertyValue)) { + errorMessage.append(String.format(" Config:%s Property:%s\n", configType, propertyName)); + containsSecretReferences = true; + } + } + } + } + if (containsSecretReferences) { + throw new InvalidTopologyException("Secret references are not allowed in blueprints, " + + "replace following properties with real passwords:\n" + errorMessage); + } + } + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java index f028a31e1d0..7166998b016 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidator.java @@ -19,7 +19,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +44,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { return; } - Set stackConfigTypes = new HashSet<>(topology.getBlueprint().getStack().getConfiguration().getAllConfigTypes()); + Set stackConfigTypes = new HashSet<>(topology.getStack().getConfiguration().getAllConfigTypes()); // remove all "valid" config types from the incoming set incomingConfigTypes.removeAll(stackConfigTypes); @@ -58,7 +57,3 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { } } } - - - - diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java similarity index 84% rename from ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java rename to ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java index c3e4b2bfa4c..41b4ad5561b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidator.java @@ -16,7 +16,10 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; /** * Performs topology validation. diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index bc76bff1e7b..1e81ef84b97 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -16,16 +16,33 @@ import java.util.List; -import org.apache.ambari.server.topology.TopologyValidator; +import javax.inject.Inject; +import javax.inject.Provider; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; import com.google.common.collect.ImmutableList; public class TopologyValidatorFactory { - List validators; - public TopologyValidatorFactory() { - validators = ImmutableList.of(new RequiredConfigPropertiesValidator(), new RequiredPasswordValidator(), new HiveServiceValidator(), - new StackConfigTypeValidator(), new UnitValidator(UnitValidatedProperty.ALL)); + private final List validators; + + @Inject + public TopologyValidatorFactory(Provider metaInfo, Configuration config) { + validators = ImmutableList.builder() + .add(new RejectUnknownStacks(metaInfo)) + .add(new RejectUnknownComponents()) + .add(new DependencyAndCardinalityValidator()) + .add(new StackConfigTypeValidator()) + .add(new GplPropertiesValidator(config)) + .add(new SecretReferenceValidator()) + .add(new RequiredConfigPropertiesValidator()) + .add(new RequiredPasswordValidator()) + .add(new HiveServiceValidator()) + .add(new NameNodeHighAvailabilityValidator()) + .add(new UnitValidator(UnitValidatedProperty.ALL)) + .build(); } public TopologyValidator createConfigurationValidatorChain() { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java index 31d5275d220..18e3913361a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java @@ -27,7 +27,6 @@ import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyValidator; /** * I validate the unit of properties by checking if it matches to the stack defined unit. @@ -42,7 +41,7 @@ public UnitValidator(Set propertiesToBeValidated) { @Override public void validate(ClusterTopology topology) throws InvalidTopologyException { - StackDefinition stack = topology.getBlueprint().getStack(); + StackDefinition stack = topology.getStack(); validateConfig(topology.getConfiguration().getFullProperties(), stack); for (HostGroupInfo hostGroup : topology.getHostGroupInfo().values()) { validateConfig(hostGroup.getConfiguration().getFullProperties(), stack); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java index 3d9a013d561..6bc2c93bf4d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/AgentResourceTest.java @@ -80,8 +80,14 @@ import org.apache.ambari.server.state.scheduler.RequestExecutionImpl; import org.apache.ambari.server.state.stack.OsFamily; import org.apache.ambari.server.state.svccomphost.ServiceComponentHostImpl; +import org.apache.ambari.server.topology.ComponentResolver; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; +import org.apache.ambari.server.topology.StackComponentResolver; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.easymock.EasyMock; @@ -335,6 +341,9 @@ protected void configure() { bind(PersistedState.class).toInstance(createNiceMock(PersistedState.class)); bind(RoleCommandOrderProvider.class).to(CachedRoleCommandOrderProvider.class); bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); + bind(ComponentResolver.class).to(StackComponentResolver.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); + bind(StackFactory.class).to(DefaultStackFactory.class); } private void installDependencies() { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java index 7a1ce8b2175..0cb0e88b373 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java @@ -19,10 +19,8 @@ package org.apache.ambari.server.api.query.render; import static java.util.stream.Collectors.toList; -import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.createStrictMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -63,10 +61,14 @@ import org.apache.ambari.server.controller.internal.ResourceImpl; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.controller.spi.ClusterController; +import org.apache.ambari.server.controller.spi.NoSuchParentResourceException; +import org.apache.ambari.server.controller.spi.NoSuchResourceException; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.DesiredConfig; @@ -102,6 +104,7 @@ public class ClusterBlueprintRendererTest { private static final ClusterTopology topology = createNiceMock(ClusterTopology.class); + private static final ClusterTopology topologyWithKerberos = createNiceMock(ClusterTopology.class); private static final ClusterController clusterController = createNiceMock(ClusterControllerImpl.class); private static final AmbariContext ambariContext = createNiceMock(AmbariContext.class); @@ -124,9 +127,14 @@ public class ClusterBlueprintRendererTest { private static final Configuration clusterConfig = new Configuration(clusterProps, clusterAttributes); public static final StackId STACK_ID = new StackId("HDP", "1.3.3"); + private static final ResourceProvider artifactResourceProvider = createNiceMock(ResourceProvider.class); + private static final Resource artifactResource = createNiceMock(Resource.class); @Before public void setup() throws Exception { + PowerMock.mockStatic(AmbariContext.class); + expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); + expect(AmbariContext.getController()).andReturn(controller).anyTimes(); Map clusterTypeProps = new HashMap<>(); clusterProps.put("test-type-one", clusterTypeProps); @@ -163,11 +171,10 @@ public void setup() throws Exception { groupInfoMap.put("host_group_1", group1Info); groupInfoMap.put("host_group_2", group2Info); - expect(topology.isNameNodeHAEnabled()).andReturn(false).anyTimes(); - expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + setupTopology(topology, groupInfoMap); + expect(topology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); + setupTopology(topologyWithKerberos, groupInfoMap); + expect(topologyWithKerberos.isClusterKerberosEnabled()).andReturn(true).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); expect(blueprint.getHostGroup("host_group_1")).andReturn(group1).anyTimes(); @@ -182,11 +189,6 @@ public void setup() throws Exception { expect(group1.getComponents()).andReturn(group1Components).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); - expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); - expect(topology.getClusterId()).andReturn(1L).anyTimes(); - PowerMock.mockStatic(AmbariServer.class); - expect(AmbariServer.getController()).andReturn(controller).anyTimes(); - PowerMock.replay(AmbariServer.class); expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes(); expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes(); expect(controller.getClusters()).andReturn(clusters).anyTimes(); @@ -195,64 +197,40 @@ public void setup() throws Exception { properties.add("core-site/hadoop.security.auth_to_local"); expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes(); expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes(); - replay(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); - } - - private void setupMocksForKerberosEnabledCluster() throws Exception { - AmbariContext ambariContext = createNiceMock(AmbariContext.class); - expect(ambariContext.getClusterName(anyLong())).andReturn("clusterName").anyTimes(); - - PowerMock.mockStatic(AmbariContext.class); - expect(AmbariContext.getClusterController()).andReturn(clusterController).anyTimes(); - expect(AmbariContext.getController()).andReturn(controller).anyTimes(); - - reset(topology); - - HostGroupInfo group1Info = new HostGroupInfo("host_group_1"); - group1Info.addHost("host1"); - group1Info.setConfiguration(emptyConfiguration); - HostGroupInfo group2Info = new HostGroupInfo("host_group_2"); - Map groupInfoMap = new HashMap<>(); - group2Info.addHosts(Arrays.asList("host2", "host3")); - group2Info.setConfiguration(emptyConfiguration); - groupInfoMap.put("host_group_1", group1Info); - groupInfoMap.put("host_group_2", group2Info); + setupKerberosDescriptorArtifact(); - expect(topology.isNameNodeHAEnabled()).andReturn(false).anyTimes(); - expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); - expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); - expect(topology.getClusterId()).andReturn(new Long(1)).anyTimes(); - expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); - expect(topology.isClusterKerberosEnabled()).andReturn(true).anyTimes(); - - ResourceProvider resourceProvider = createStrictMock(ResourceProvider.class); - expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(resourceProvider).once(); - - Resource resource = createStrictMock(Resource.class); - Set result = Collections.singleton(resource); + PowerMock.replay(AmbariContext.class); + replay(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + } - expect(resourceProvider.getResources((Request) anyObject(Request.class), - (Predicate) anyObject(Predicate.class))).andReturn(result).once(); + @After + public void tearDown() { + verify(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + reset(topology, topologyWithKerberos, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor, clusterController, artifactResource, artifactResourceProvider); + PowerMock.reset(AmbariContext.class); + } + private void setupKerberosDescriptorArtifact() throws SystemException, UnsupportedPropertyException, NoSuchResourceException, NoSuchParentResourceException { Map> resourcePropertiesMap = new HashMap<>(); resourcePropertiesMap.put(ArtifactResourceProvider.ARTIFACT_DATA_PROPERTY, Collections.emptyMap()); Map propertiesMap = new HashMap<>(); propertiesMap.put("testProperty", "testValue"); resourcePropertiesMap.put(ArtifactResourceProvider.ARTIFACT_DATA_PROPERTY + "/properties", propertiesMap); - expect(resource.getPropertiesMap()).andReturn(resourcePropertiesMap).once(); - - PowerMock.replay(AmbariContext.class); - replay(ambariContext, topology, clusterController, resource, resourceProvider); + expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).anyTimes(); + expect(artifactResourceProvider.getResources(anyObject(Request.class), anyObject(Predicate.class))).andReturn(Collections.singleton(artifactResource)).anyTimes(); + expect(artifactResource.getPropertiesMap()).andReturn(resourcePropertiesMap).anyTimes(); } - - @After - public void tearDown() { - verify(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); - reset(topology, blueprint, stack, group1, group2, ambariContext, clusters, controller, kerberosHelper, cluster, kerberosDescriptor); + private void setupTopology(ClusterTopology topology, Map groupInfoMap) { + expect(topology.getConfiguration()).andReturn(clusterConfig).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(topology.getClusterId()).andReturn(1L).anyTimes(); } @Test @@ -443,13 +421,10 @@ public void testFinalizeProperties__instance_noComponentNode() { @Test public void testFinalizeResult_kerberos() throws Exception{ - - setupMocksForKerberosEnabledCluster(); - Result result = new ResultImpl(true); createClusterResultTree(result.getResultTree()); - ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topology); + ClusterBlueprintRenderer renderer = new TestBlueprintRenderer(topologyWithKerberos); Result blueprintResult = renderer.finalizeResult(result); TreeNode blueprintTree = blueprintResult.getResultTree(); @@ -465,6 +440,7 @@ public void testFinalizeResult_kerberos() throws Exception{ assertEquals(STACK_ID.getStackVersion(), properties.get("Blueprints").get("stack_version")); Map securityProperties = (Map) properties.get("Blueprints").get("security"); + assertNotNull(securityProperties); assertEquals("KERBEROS", securityProperties.get("type")); assertNotNull(((Map) securityProperties.get("kerberos_descriptor")).get("properties")); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java index 10fde6fa39f..2abc1522799 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java @@ -87,16 +87,16 @@ public void testAdviseConfiguration() throws StackAdvisorException, Configuratio expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -126,16 +126,16 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApply() throws StackAdvi expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -166,16 +166,16 @@ public void testAdviseConfigurationWithOnlyStackDefaultsApplyWhenNoUserInputForD expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -204,16 +204,16 @@ public void testAdviseConfigurationWith_ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); expect(clusterTopology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.ALWAYS_APPLY_DONT_OVERRIDE_CUSTOM_VALUES).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); - expect(blueprint.isValidConfigType("core-site")).andReturn(true).anyTimes(); + expect(clusterTopology.isValidConfigType("core-site")).andReturn(true).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(createRecommendationResponse()); expect(configuration.getFullProperties()).andReturn(props).anyTimes(); @@ -240,15 +240,15 @@ public void testAdviseConfigurationWhenConfigurationRecommendFails() throws Stac expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes(); expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andThrow(new StackAdvisorException("ex")); expect(configuration.getFullProperties()).andReturn(props); @@ -273,14 +273,14 @@ public void testAdviseConfigurationWhenConfigurationRecommendHasInvalidResponse( expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes(); expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes(); expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getStackId()).andReturn(STACK_ID).anyTimes(); expect(stack.getVersion()).andReturn(STACK_ID.getStackVersion()).anyTimes(); expect(stack.getName()).andReturn(STACK_ID.getStackName()).anyTimes(); expect(stack.getConfiguration(services)).andReturn(createStackDefaults()).anyTimes(); expect(stack.getServices(STACK_ID)).andReturn(services).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(clusterTopology.getServices()).andReturn(services).anyTimes(); expect(blueprint.getHostGroups()).andReturn(createHostGroupMap()).anyTimes(); expect(hostGroup.getComponentNames()).andReturn(Arrays.asList("comp1", "comp2")).anyTimes(); expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(new RecommendationResponse()); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 6e373d8207e..9c6f495d091 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1065,9 +1065,9 @@ public void testCreateServiceComponentWithInvalidRequest() try { set1.clear(); ServiceComponentRequest rInvalid1 = - new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", null); + new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", State.INSTALLED.name()); ServiceComponentRequest rInvalid2 = - new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", null); + new ServiceComponentRequest(cluster1, serviceGroupName, "HDFS", "HDFS_CLIENT", State.STARTED.name()); set1.add(rInvalid1); set1.add(rInvalid2); ComponentResourceProviderTest.createComponents(controller, set1); @@ -3173,7 +3173,7 @@ public void testServiceUpdateInvalidRequest() throws Exception { req1 = new ServiceRequest(cluster1, serviceGroupName, serviceName1, repositoryVersion02.getId(), State.INSTALLED.toString(), null); req2 = new ServiceRequest(cluster1, serviceGroupName, serviceName1, repositoryVersion02.getId(), - State.INSTALLED.toString(), null); + State.STARTED.toString(), null); reqs.add(req1); reqs.add(req2); ServiceResourceProviderTest.updateServices(controller, reqs, mapRequestProps, true, false); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java deleted file mode 100644 index 341e889fdad..00000000000 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BaseBlueprintProcessorTest.java +++ /dev/null @@ -1,550 +0,0 @@ -package org.apache.ambari.server.controller.internal; - -import static org.junit.Assert.assertEquals; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.ambari.server.state.DependencyInfo; -import org.apache.ambari.server.state.StackInfo; -import org.junit.Test; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -@SuppressWarnings("unchecked") -public class BaseBlueprintProcessorTest { - - //todo: Move these tests to the correct location. - //todo: BaseBluprintProcess no longer exists. - @Test - public void testStackRegisterConditionalDependencies() throws Exception { - StackInfo stackInfo = new StackInfo(); - - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(stackInfo) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 5, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoHCAT() throws Exception { - // test dependencies - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoYarnClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoTezClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoMapReduceClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo oozieClientDependency = new TestDependencyInfo( - "OOZIE/OOZIE_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(oozieClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for OOZIE_CLIENT", - "OOZIE", - testStack.getDependencyConditionalServiceMap().get( - oozieClientDependency)); - } - - @Test - public void testStackRegisterConditionalDependenciesNoOozieClient() - throws Exception { - // test dependencies - final DependencyInfo hCatDependency = new TestDependencyInfo("HIVE/HCAT"); - final DependencyInfo yarnClientDependency = new TestDependencyInfo( - "YARN/YARN_CLIENT"); - final DependencyInfo tezClientDependency = new TestDependencyInfo( - "TEZ/TEZ_CLIENT"); - final DependencyInfo mapReduceTwoClientDependency = new TestDependencyInfo( - "YARN/MAPREDUCE2_CLIENT"); - - // create stack for testing - Stack testStack = new Stack(new StackInfo()) { - @Override - public Collection getDependenciesForComponent( - String component) { - // simulate the dependencies in a given stack by overriding this method - if (component.equals("FAKE_MONITORING_SERVER")) { - Set setOfDependencies = new HashSet<>(); - - setOfDependencies.add(hCatDependency); - setOfDependencies.add(yarnClientDependency); - setOfDependencies.add(tezClientDependency); - setOfDependencies.add(mapReduceTwoClientDependency); - - return setOfDependencies; - } - - return Collections.emptySet(); - } - - /** - * {@inheritDoc} - */ - @Override - void registerConditionalDependencies() { - // TODO Auto-generated method stub - super.registerConditionalDependencies(); - - Map dependencyConditionalServiceMap = getDependencyConditionalServiceMap(); - Collection monitoringDependencies = getDependenciesForComponent("FAKE_MONITORING_SERVER"); - for (DependencyInfo dependency : monitoringDependencies) { - if (dependency.getComponentName().equals("HCAT")) { - dependencyConditionalServiceMap.put(dependency, "HIVE"); - } else if (dependency.getComponentName().equals("OOZIE_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "OOZIE"); - } else if (dependency.getComponentName().equals("YARN_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "YARN"); - } else if (dependency.getComponentName().equals("TEZ_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "TEZ"); - } else if (dependency.getComponentName().equals("MAPREDUCE2_CLIENT")) { - dependencyConditionalServiceMap.put(dependency, "MAPREDUCE2"); - } - } - } - - }; - - assertEquals("Initial conditional dependency map should be empty", 0, - testStack.getDependencyConditionalServiceMap().size()); - - testStack.registerConditionalDependencies(); - - assertEquals("Set of conditional service mappings is an incorrect size", 4, - testStack.getDependencyConditionalServiceMap().size()); - - assertEquals("Incorrect service dependency for HCAT", "HIVE", - testStack.getDependencyConditionalServiceMap().get(hCatDependency)); - assertEquals( - "Incorrect service dependency for YARN_CLIENT", - "YARN", - testStack.getDependencyConditionalServiceMap().get(yarnClientDependency)); - assertEquals("Incorrect service dependency for TEZ_CLIENT", "TEZ", - testStack.getDependencyConditionalServiceMap().get(tezClientDependency)); - assertEquals( - "Incorrect service dependency for MAPREDUCE2_CLIENT", - "MAPREDUCE2", - testStack.getDependencyConditionalServiceMap().get( - mapReduceTwoClientDependency)); - } - - /** - * Convenience class for easier setup/initialization of dependencies for unit - * testing. - */ - private static class TestDependencyInfo extends DependencyInfo { - TestDependencyInfo(String dependencyName) { - setName(dependencyName); - } - } -} \ No newline at end of file diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 23283702dfa..84e3593b992 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -18,7 +18,13 @@ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.ALWAYS_APPLY; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.NEVER_APPLY; +import static org.apache.ambari.server.topology.ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -56,6 +62,7 @@ import org.apache.ambari.server.topology.AdvisedConfiguration; import org.apache.ambari.server.topology.AmbariContext; import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.BlueprintBasedClusterProvisionRequest; import org.apache.ambari.server.topology.Cardinality; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ClusterTopologyImpl; @@ -66,7 +73,9 @@ import org.apache.ambari.server.topology.HostGroupImpl; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.apache.ambari.server.topology.TopologyRequest; +import org.apache.ambari.server.topology.ResolvedComponent; +import org.apache.ambari.server.topology.SecurityConfiguration; +import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.commons.lang.StringUtils; import org.easymock.EasyMock; import org.easymock.EasyMockRule; @@ -137,33 +146,34 @@ public class BlueprintConfigurationProcessorTest extends EasyMockSupport { private Cluster cluster; @Mock - private TopologyRequest topologyRequestMock; + private ProvisionClusterRequest topologyRequestMock; @Mock(type = MockType.NICE) private ConfigHelper configHelper; + @Mock(type = MockType.NICE) + private SecurityConfigurationFactory securityFactory; + @Before public void init() throws Exception { - expect(bp.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(bp.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(bp.getName()).andReturn("test-bp").anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).atLeastOnce(); expect(stack.getVersion()).andReturn(STACK_VERSION).atLeastOnce(); // return false for all components since for this test we don't care about the value - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(Collections.emptyMap()).anyTimes(); - expect(serviceInfo.getRequiredProperties()).andReturn( - Collections.emptyMap()).anyTimes(); + expect(serviceInfo.getRequiredProperties()).andReturn(Collections.emptyMap()).anyTimes(); expect(serviceInfo.getRequiredServices()).andReturn(Collections.emptyList()).anyTimes(); setupGetServiceForComponentExpectations(); expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - Set emptySet = Collections.emptySet(); - expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); + expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(Collections.emptySet()).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); expect(configHelper.getDefaultStackProperties( @@ -289,7 +299,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -337,14 +347,14 @@ public void testDoUpdateForBlueprintExport_FilterProperties() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); assertEquals(properties.size(), 3); - assertEquals(((Map) properties.get("kerberos-env")).size(), 0); - assertEquals(((Map) properties.get("krb5-conf")).size(), 0); - assertEquals(((Map) properties.get("tez-site")).size(), 0); + assertEquals((properties.get("kerberos-env")).size(), 0); + assertEquals((properties.get("krb5-conf")).size(), 0); + assertEquals((properties.get("tez-site")).size(), 0); } @Test @@ -371,7 +381,7 @@ public void testDoUpdateForBlueprintExportRangerHAPolicyMgrExternalUrlProperty() Collection hostGroups = ImmutableSet.of(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -415,7 +425,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty_specifiedInParentC hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -463,7 +473,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty_hostGroupConfigura hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -498,7 +508,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty__withPort() throws hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -531,7 +541,7 @@ public void testDoUpdateForBlueprintExport_SingleHostProperty__ExternalReference hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -577,7 +587,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty() throws Exception hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -624,7 +634,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty__WithPorts() throws hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -679,7 +689,7 @@ public void testDoUpdateForBlueprintExport_MultiHostProperty__YAML() throws Exce hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -722,7 +732,7 @@ public void testDoUpdateForBlueprintExport_DBHostProperty() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -755,7 +765,7 @@ public void testDoUpdateForBlueprintExport_DBHostProperty__External() throws Exc hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -807,7 +817,7 @@ public void testDoUpdateForBlueprintExport_PasswordFilterApplied() throws Except hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -890,7 +900,7 @@ public void testFalconConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -927,7 +937,7 @@ public void testTezConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -939,8 +949,6 @@ public void testTezConfigExport() throws Exception { * There is no support currently for deploying a fully Kerberized * cluster with Blueprints. This test verifies the current treatment * of Kerberos-related properties in a Blueprint export. - * - * @throws Exception */ @Test public void testKerberosConfigExport() throws Exception { @@ -978,7 +986,7 @@ public void testKerberosConfigExport() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1042,7 +1050,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabled() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1115,7 +1123,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledPrimaryNamePreferen Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1175,7 +1183,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledNameServiceProperti Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1211,7 +1219,7 @@ public void testDoNameNodeHighAvailabilityExportWithHANotEnabled() throws Except Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1274,7 +1282,7 @@ public void testDoNameNodeHighAvailabilityExportWithHAEnabledMultipleServices() Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1351,7 +1359,7 @@ public void testYarnConfigExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1417,7 +1425,7 @@ public void testYarnConfigExportedWithDefaultZeroHostAddress() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1493,7 +1501,7 @@ public void testHDFSConfigExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -1580,7 +1588,7 @@ public void testHiveConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1673,7 +1681,7 @@ public void testHiveConfigExportedMultipleHiveMetaStoreServers() throws Exceptio hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1769,7 +1777,7 @@ public void testOozieConfigExported() throws Exception { BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").remove("oozie.service.JPAService.jdbc.url"); } - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -1832,7 +1840,7 @@ public void testOozieJDBCPropertiesNotRemoved() throws Exception { expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); @@ -1871,7 +1879,7 @@ public void testOozieJDBCPropertyAddedToSingleHostMapDuringImport() throws Excep expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor blueprintConfigurationProcessor = new BlueprintConfigurationProcessor(topology); assertTrue(BlueprintConfigurationProcessor.singleHostTopologyUpdaters.get("oozie-site").containsKey("oozie.service.JPAService.jdbc.url")); @@ -1934,7 +1942,7 @@ public void testZookeeperConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2014,7 +2022,7 @@ public void testKnoxSecurityConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2063,7 +2071,7 @@ public void testKafkaConfigExported() throws Exception { hostGroups.add(group); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2102,7 +2110,7 @@ public void testPropertyWithUndefinedHostisExported() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -2144,7 +2152,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue() thro hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set configTypesUpdated = @@ -2200,7 +2208,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_provid hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2247,7 +2255,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_hostGr hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2295,7 +2303,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue_BPHost hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); // todo: set as BP hostgroup topology.getHostGroupInfo().get("group2").getConfiguration().setParentConfiguration(group2BPConfig); @@ -2334,7 +2342,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MissingComponent() expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); //todo: should throw a checked exception, not the exception expected by the api @@ -2377,7 +2385,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleMatchingHos expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); try { @@ -2421,7 +2429,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MultipleAppTimeline expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2457,7 +2465,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__MissingOptionalComp expect(stack.getCardinality("APP_TIMELINE_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2489,7 +2497,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__defaultValue__WithP hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("core-site").get("fs.defaultFS"); @@ -2534,7 +2542,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues() thro hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("hbase-site").get("hbase.zookeeper.quorum"); @@ -2591,7 +2599,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___with hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedVal = topology.getConfiguration().getFullProperties().get("webhcat-site").get("templeton.zookeeper.hosts"); @@ -2642,7 +2650,7 @@ public void testMultipleHostTopologyUpdater__localhost__singleHost() throws Exce hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2676,7 +2684,7 @@ public void testMultipleHostTopologyUpdater__localhost__singleHostGroup() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2711,7 +2719,7 @@ public void testMultipleHostTopologyUpdater__hostgroup__singleHostGroup() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor.MultipleHostTopologyUpdater mhtu = new BlueprintConfigurationProcessor.MultipleHostTopologyUpdater(component1); String newValue = mhtu.updateForClusterCreate(propertyName, originalValue, properties, topology); @@ -2731,7 +2739,7 @@ public void testDoUpdateForClusterVerifyRetrySettingsDefault() throws Exception Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); @@ -2771,7 +2779,7 @@ public void testDoUpdateForClusterVerifyRetrySettingsCustomized() throws Excepti Configuration clusterConfig = new Configuration(configProperties, Collections.emptyMap()); TestHostGroup testHostGroup = new TestHostGroup("test-host-group-one", Collections.emptySet(), Collections.emptySet()); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup)); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, Collections.singleton(testHostGroup), NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); @@ -2859,7 +2867,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabledSpecifyingHostNamesDirect expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2934,7 +2942,7 @@ public void testHiveConfigClusterUpdateCustomValueSpecifyingHostNamesMetaStoreHA hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -2983,7 +2991,7 @@ public void testHiveConfigClusterUpdateSpecifyingHostNamesHiveServer2HA() throws expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3047,7 +3055,7 @@ public void testHiveConfigClusterUpdateUsingExportedNamesHiveServer2HA() throws expect(stack.getCardinality("HIVE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3108,7 +3116,7 @@ private void testHiveMetastoreHA(String separator) throws InvalidTopologyExcepti hostGroups.add(new TestHostGroup("host_group_" + i, components, Collections.singleton(hostNames[i]))); } - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); String updatedValue = webHCatSiteProperties.get(propertyKey); @@ -3162,7 +3170,7 @@ public void testHiveInteractiveLlapZookeeperConfigExported() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3216,7 +3224,7 @@ public void testOozieConfigClusterUpdateHAEnabledSpecifyingHostNamesDirectly() t expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3282,7 +3290,7 @@ public void testOozieHAEnabledExport() throws Exception { expect(stack.getCardinality("OOZIE_SERVER")).andReturn(new Cardinality("1+")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3336,7 +3344,7 @@ public void testYarnHighAvailabilityConfigClusterUpdateSpecifyingHostNamesDirect expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3423,7 +3431,7 @@ public void testYarnHighAvailabilityExport() throws Exception { expect(stack.getCardinality("RESOURCEMANAGER")).andReturn(new Cardinality("1-2")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForBlueprintExport(); @@ -3506,7 +3514,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURLSpecifyingHostNamesDirect hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3555,7 +3563,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__defaultValues___YAML hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3606,7 +3614,7 @@ public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__defaultValues_ hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3656,7 +3664,7 @@ public void testDoUpdateForClusterCreate_Storm_Nimbus_HA_Enabled__FQDN_ValuesSpe hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3691,7 +3699,7 @@ public void testDoUpdateForClusterCreate_MProperty__defaultValues() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3723,7 +3731,7 @@ public void testDoUpdateForClusterCreate_MProperty__missingM() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3755,7 +3763,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue() thr hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3787,7 +3795,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_Using hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3819,7 +3827,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue_WithP hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3851,7 +3859,7 @@ public void testDoUpdateForClusterCreate_SingleHostProperty__exportedValue__With hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3897,7 +3905,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues() thr hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -3955,7 +3963,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___wit hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4013,7 +4021,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___wit hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4054,7 +4062,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty_exportedValues_withPo Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4110,7 +4118,7 @@ public void testDoUpdateForClusterCreate_MultiHostProperty__exportedValues___YAM hostGroups.add(group3); hostGroups.add(group4); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4184,7 +4192,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__defaultValue() throws E hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4220,7 +4228,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__exportedValue() throws hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4253,7 +4261,7 @@ public void testDoUpdateForClusterCreate_DBHostProperty__external() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4268,17 +4276,17 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")); expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.emptySet()); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4296,7 +4304,7 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4312,16 +4320,16 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4337,7 +4345,7 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4352,14 +4360,14 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyObject(String.class), anyObject(String.class))).andReturn(Collections.emptyMap()).anyTimes(); // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); Map> properties = new HashMap<>(); Map typeProps = new HashMap<>(); @@ -4379,7 +4387,7 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4395,7 +4403,7 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); // customized stack calls for this test only Set excludedConfigTypes = new HashSet<>(); @@ -4407,10 +4415,10 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); // simulate the case where the STORM service has been removed manually from the stack definitions expect(stack.getServiceForConfigType("storm-site")).andThrow(new IllegalArgumentException("TEST: Configuration not found in stack definitions!")); - expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); - expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); - expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); - expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE").anyTimes(); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4428,7 +4436,7 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4466,7 +4474,7 @@ public void testFalconConfigClusterUpdate() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -4507,7 +4515,7 @@ public void testFalconConfigClusterUpdateDefaultConfig() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -4543,7 +4551,7 @@ public void testHiveConfigClusterUpdateCustomValue() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4599,7 +4607,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4621,7 +4629,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4678,7 +4686,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4700,7 +4708,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4758,7 +4766,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4779,7 +4787,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4846,7 +4854,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4868,7 +4876,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4916,7 +4924,7 @@ Set getDependsOnProperties() { // defaults from init() method that we need expect(stack.getName()).andReturn("testStack").anyTimes(); expect(stack.getVersion()).andReturn("1").anyTimes(); - expect(stack.isMasterComponent((String) anyObject())).andReturn(false).anyTimes(); + expect(stack.isMasterComponent(anyObject())).andReturn(false).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -4937,7 +4945,7 @@ Set getDependsOnProperties() { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -4975,7 +4983,7 @@ public void testHiveConfigClusterUpdateDefaultValue() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5028,7 +5036,7 @@ public void testAtlas() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5092,7 +5100,7 @@ public void testHiveConfigClusterUpdateExportedHostGroupValue() throws Exception Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level cluster config update method @@ -5132,7 +5140,7 @@ public void testStormAndKafkaConfigClusterUpdateWithoutGangliaServer() throws Ex expect(stack.getCardinality("GANGLIA_SERVER")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5180,7 +5188,7 @@ public void testStormandKafkaConfigClusterUpdateWithGangliaServer() throws Excep Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5275,7 +5283,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabled() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set updatedConfigTypes = @@ -5394,7 +5402,7 @@ public void testDoUpdateForClusterWithNameNodeHANotEnabled() throws Exception { expect(stack.getCardinality("NAMENODE")).andReturn(new Cardinality("1-2")).anyTimes(); expect(stack.getCardinality("SECONDARY_NAMENODE")).andReturn(new Cardinality("1")).anyTimes(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); Set updatedConfigTypes = @@ -5463,7 +5471,7 @@ public void testDoUpdateForClusterWithNameNodeHAEnabledAndActiveNodeSet() throws Collection hostGroups = new ArrayList<>(); hostGroups.add(group); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -5610,7 +5618,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURL() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5652,7 +5660,7 @@ public void testHDFSConfigClusterUpdateQuorumJournalURL_UsingMinusSymbolInHostNa hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5703,7 +5711,7 @@ public void testHadoopHaNameNode() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -5743,7 +5751,7 @@ public void testGetRequiredHostGroups___validComponentCountOfZero() throws Excep hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5776,7 +5784,7 @@ public void testGetRequiredHostGroups___invalidComponentCountOfZero() throws Exc hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5811,7 +5819,7 @@ public void testGetRequiredHostGroups___multipleGroups() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); // call top-level export method @@ -5855,7 +5863,7 @@ public void testAllDefaultUserAndGroupProxyPropertiesSet() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5900,7 +5908,7 @@ public void testRelevantDefaultUserAndGroupProxyPropertiesSet() throws Exception Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5944,7 +5952,7 @@ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided() throws Ex Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -5987,7 +5995,7 @@ public void testDefaultUserAndGroupProxyPropertiesSetWhenNotProvided2() throws E Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6031,7 +6039,7 @@ public void testHiveWithoutAtlas() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6122,7 +6130,7 @@ private void validateAtlasHivePropertiesForTestCase(Map hostGroups = Collections.singletonList(group1); - ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology1 = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology1); configProcessor.doUpdateForClusterCreate(); @@ -6170,7 +6178,7 @@ public void testAtlasHivePropertiesWithHTTPS() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6202,7 +6210,7 @@ public void testStormAmsPropertiesDefault() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6233,7 +6241,7 @@ public void testStormAmsPropertiesUserDefinedReporter() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6264,7 +6272,7 @@ public void testKafkaAmsProperties() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6296,7 +6304,7 @@ public void testKafkaAmsPropertiesMultipleReporters() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -6329,7 +6337,6 @@ public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception Collection hgComponents = new HashSet<>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); - hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection hgComponents2 = new HashSet<>(); @@ -6347,15 +6354,15 @@ public void testRecommendConfiguration_applyStackDefaultsOnly() throws Exception Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ONLY_STACK_DEFAULTS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); - expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); + expect(stack.getServiceForComponent(anyString())).andReturn("HDFS").anyTimes(); + expect(stack.getConfiguration(ImmutableSet.of("HDFS"))).andReturn(createStackDefaults()).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -6387,7 +6394,6 @@ public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly Collection hgComponents = new HashSet<>(); hgComponents.add("NAMENODE"); hgComponents.add("SECONDARY_NAMENODE"); - hgComponents.add("RESOURCEMANAGER"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents, Collections.singleton("testhost")); Collection hgComponents2 = new HashSet<>(); @@ -6407,15 +6413,15 @@ public void testRecommendConfiguration_EmptyConfiguration_applyStackDefaultsOnly Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ONLY_STACK_DEFAULTS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ONLY_STACK_DEFAULTS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); reset(stack); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); - expect(stack.getConfiguration(bp.getServices())).andReturn(createStackDefaults()).anyTimes(); + expect(stack.getServiceForComponent(anyString())).andReturn("HDFS").anyTimes(); + expect(stack.getConfiguration(ImmutableSet.of("HDFS"))).andReturn(createStackDefaults()).anyTimes(); Set emptySet = Collections.emptySet(); expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); @@ -6475,9 +6481,8 @@ public void testRecommendConfiguration_applyAlways() throws Exception { Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, ALWAYS_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.ALWAYS_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // WHEN Set configTypes = configProcessor.doUpdateForClusterCreate(); @@ -6530,9 +6535,8 @@ public void testRecommendConfiguration_neverApply() throws Exception { Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); topology.getAdvisedConfigurations().putAll(createAdvisedConfigMap()); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // WHEN configProcessor.doUpdateForClusterCreate(); @@ -6571,7 +6575,7 @@ public void testRangerAdminProperties() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6607,7 +6611,7 @@ public void testRangerAdminProperties_defaults() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6644,7 +6648,7 @@ public void testRangerAdminProperties_HA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6699,7 +6703,7 @@ public void testRangerEnv_defaults() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6758,7 +6762,7 @@ public void testRangerEnv_defaults_NO_HDFS() throws Exception { Collection hostGroups = Lists.newArrayList(group1);//, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6824,7 +6828,7 @@ public void testRangerEnv() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6898,7 +6902,7 @@ public void testRangerEnvWithHdfsHA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -6964,7 +6968,7 @@ public void testRangerEnvBlueprintExport() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7036,7 +7040,7 @@ public void testRangerEnvExportBlueprintWithHdfsHA() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7087,7 +7091,7 @@ public void testRangerKmsServerProperties() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7127,7 +7131,7 @@ public void testRangerKmsServerProperties_default() throws Exception { Collection hostGroups = Collections.singleton(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7172,7 +7176,7 @@ public void testHdfsWithRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7226,7 +7230,7 @@ public void testHdfsWithNoRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); @@ -7271,7 +7275,7 @@ public void testHdfsWithRangerKmsServer_default() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7317,7 +7321,7 @@ public void testHdfsWithRangerKmsServer__multiple_hosts__localhost() throws Exce Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7371,7 +7375,7 @@ public void testHdfsWithRangerKmsServer__multiple_hosts__hostgroup() throws Exce Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7427,7 +7431,7 @@ public void testResolutionOfDRPCServerAndNN() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7471,7 +7475,7 @@ public void testHadoopWithRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7515,7 +7519,7 @@ public void testHadoopWithNoRangerKmsServer() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); @@ -7560,7 +7564,7 @@ public void testHadoopWithRangerKmsServer_default() throws Exception { Collection hostGroups = Lists.newArrayList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); // When @@ -7815,7 +7819,7 @@ public void testHawqConfigClusterUpdate() throws Exception { hostGroups.add(group2); hostGroups.add(group3); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -7847,7 +7851,7 @@ public void testHawqNonHaConfigClusterUpdate() throws Exception { Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor updater = new BlueprintConfigurationProcessor(topology); updater.doUpdateForClusterCreate(); @@ -7870,7 +7874,7 @@ public void testDoUpdateForBlueprintExport_NonTopologyProperty__AtlasClusterName Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); Long clusterId = topology.getClusterId(); Map typeProps = new HashMap<>(); typeProps.put("atlas.cluster.name", String.valueOf(clusterId)); @@ -7900,8 +7904,7 @@ public void testDoUpdateForBlueprintExport_NonTopologyProperty() throws Exceptio Collection hostGroups = new HashSet<>(); hostGroups.add(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); - Long clusterId = topology.getClusterId(); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); Map hiveSiteProps = new HashMap<>(); hiveSiteProps.put("hive.exec.post.hooks", someString); @@ -7938,8 +7941,8 @@ public void druidProperties() throws Exception { properties.put("druid-common", druidCommon); Map> parentProperties = new HashMap<>(); - Configuration parentClusterConfig = new Configuration(parentProperties, Collections.>>emptyMap()); - Configuration clusterConfig = new Configuration(properties, Collections.>>emptyMap(), parentClusterConfig); + Configuration parentClusterConfig = new Configuration(parentProperties, Collections.emptyMap()); + Configuration clusterConfig = new Configuration(properties, Collections.emptyMap(), parentClusterConfig); Collection hgComponents1 = Sets.newHashSet("DRUID_COORDINATOR"); TestHostGroup group1 = new TestHostGroup("group1", hgComponents1, Collections.singleton("host1")); @@ -7949,7 +7952,7 @@ public void druidProperties() throws Exception { Collection hostGroups = Arrays.asList(group1, group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -7979,7 +7982,7 @@ public void testAmsPropertiesDefault() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8009,7 +8012,7 @@ public void testAmsPropertiesSpecialAddress() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8044,7 +8047,7 @@ public void testAmsPropertiesSpecialAddressMultipleCollectors() throws Exception hostGroups.add(group1); hostGroups.add(group2); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8079,8 +8082,8 @@ public void testStackPasswordPropertyFilter() throws Exception { hostGroups.add(group1); hostGroups.add(group2); - expect(stack.isPasswordProperty((String) anyObject(), (String) anyObject(), (String) anyObject())).andReturn(true).once(); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + expect(stack.isPasswordProperty(anyObject(), anyObject(), anyObject())).andReturn(true).once(); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForBlueprintExport(); @@ -8159,7 +8162,7 @@ public void testValuesTrimming() throws Exception { Collection hostGroups = Collections.singletonList(group1); - ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups); + ClusterTopology topology = createClusterTopology(bp, clusterConfig, hostGroups, NEVER_APPLY); BlueprintConfigurationProcessor configProcessor = new BlueprintConfigurationProcessor(topology); configProcessor.doUpdateForClusterCreate(); @@ -8212,15 +8215,15 @@ private Configuration createStackDefaults() { } private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration configuration, - Collection hostGroups) + Collection hostGroups, ConfigRecommendationStrategy recommendation) throws InvalidTopologyException { replay(stack, serviceInfo, ambariContext, configHelper, controller, kerberosHelper, kerberosDescriptor, clusters, cluster); Map hostGroupInfo = new HashMap<>(); - Collection allServices = new HashSet<>(); Map allHostGroups = new HashMap<>(); + Map> resolvedComponents = new HashMap<>(); for (TestHostGroup hostGroup : hostGroups) { HostGroupInfo groupInfo = new HostGroupInfo(hostGroup.name); @@ -8228,45 +8231,47 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration //todo: HG configs groupInfo.setConfiguration(hostGroup.configuration); - List componentList = new ArrayList<>(); - for (String componentName : hostGroup.components) { - componentList.add(new Component(componentName)); - } + Set components = hostGroup.components.stream() + .map(name -> ResolvedComponent.builder(new Component(name)).stackId(STACK_ID).serviceType(stack.getServiceForComponent(name)).buildPartial()) + .collect(toSet()); - //create host group which is set on topology - allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", stack, - componentList, EMPTY_CONFIG, "1")); + List componentList = components.stream() + .map(ResolvedComponent::component) + .collect(toList()); + //create host group which is set on topology + allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, componentList, EMPTY_CONFIG, "1")); hostGroupInfo.put(hostGroup.name, groupInfo); - - for (String component : hostGroup.components) { - for (Map.Entry> serviceComponentsEntry : serviceComponents.entrySet()) { - if (serviceComponentsEntry.getValue().contains(component)) { - allServices.add(serviceComponentsEntry.getKey()); - } - } - } + resolvedComponents.put(hostGroup.name, components); } - expect(bp.getServices()).andReturn(allServices).anyTimes(); - for (HostGroup group : allHostGroups.values()) { expect(bp.getHostGroup(group.getName())).andReturn(group).anyTimes(); } expect(bp.getHostGroups()).andReturn(allHostGroups).anyTimes(); - expect(topologyRequestMock.getClusterId()).andReturn(1L).anyTimes(); expect(topologyRequestMock.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topologyRequestMock.getClusterId()).andReturn(1L).anyTimes(); + expect(topologyRequestMock.getConfigRecommendationStrategy()).andReturn(recommendation).anyTimes(); expect(topologyRequestMock.getConfiguration()).andReturn(configuration).anyTimes(); + expect(topologyRequestMock.getDefaultPassword()).andReturn("secret").anyTimes(); expect(topologyRequestMock.getHostGroupInfo()).andReturn(hostGroupInfo).anyTimes(); - - replay(bp, topologyRequestMock); - - ClusterTopology topology = new ClusterTopologyImpl(ambariContext, topologyRequestMock); - topology.setConfigRecommendationStrategy(ConfigRecommendationStrategy.NEVER_APPLY); - - return topology; + expect(topologyRequestMock.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(topologyRequestMock.getProvisionAction()).andReturn(ProvisionAction.INSTALL_AND_START).anyTimes(); + expect(topologyRequestMock.getSecurityConfiguration()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(topologyRequestMock.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(bp.getConfiguration()).andReturn(Configuration.createEmpty()).anyTimes(); + expect(bp.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(bp.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(securityFactory.loadSecurityConfigurationByReference(anyString())).andReturn(null).anyTimes(); + replay(bp, topologyRequestMock, securityFactory); + + BlueprintBasedClusterProvisionRequest request = new BlueprintBasedClusterProvisionRequest(ambariContext, securityFactory, bp, topologyRequestMock); + + ClusterTopologyImpl clusterTopology = new ClusterTopologyImpl(ambariContext, request, resolvedComponents); + clusterTopology.setClusterId(1L); + return clusterTopology; } private class TestHostGroup { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index dece002f9bb..aeadc5cb74a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -75,11 +75,10 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.BlueprintFactory; -import org.apache.ambari.server.topology.BlueprintValidator; -import org.apache.ambari.server.topology.InvalidTopologyException; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.ambari.server.topology.Setting; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.apache.ambari.server.utils.StageUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -90,7 +89,6 @@ /** * BlueprintResourceProvider unit tests. */ -@SuppressWarnings("unchecked") public class BlueprintResourceProviderTest { private static String BLUEPRINT_NAME = "test-blueprint"; @@ -115,7 +113,7 @@ public static void initClass() { replay(resourceProviderFactory); } - private Map>> getSettingProperties() { + private Map>> getSettingProperties() { return new HashMap<>(); } @@ -133,13 +131,12 @@ public void testCreateResources() throws Exception { Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); expect(securityFactory.createSecurityConfigurationFromRequest(null, true)).andReturn(null).anyTimes(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -208,14 +205,13 @@ public void testCreateResources_NoValidation() throws Exception { Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); requestInfoProperties.put("validate_topology", "false"); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -247,7 +243,7 @@ public void testCreateResources_NoValidation() throws Exception { verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request, managementController); } - @Test + @Test(expected = IllegalArgumentException.class) public void testCreateResources_TopologyValidationFails() throws Exception { Request request = createMock(Request.class); @@ -257,9 +253,8 @@ public void testCreateResources_TopologyValidationFails() throws Exception { // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); - expectLastCall().andThrow(new InvalidTopologyException("test")).once(); + blueprintValidator.validate(blueprint); + expectLastCall().andThrow(new IllegalArgumentException("test")).once(); expect(request.getProperties()).andReturn(setProperties); expect(request.getRequestInfoProperties()).andReturn(requestInfoProperties); @@ -275,14 +270,7 @@ public void testCreateResources_TopologyValidationFails() throws Exception { AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); ((ObservableResourceProvider)provider).addObserver(observer); - try { - provider.createResources(request); - fail("Expected exception due to topology validation error"); - } catch (IllegalArgumentException e) { - // expected - } - - verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request); + provider.createResources(request); } @@ -295,12 +283,11 @@ public void testCreateResources_withConfiguration() throws Exception { Map requestInfoProperties = getTestRequestInfoProperties(); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -332,9 +319,8 @@ public void testCreateResources_withConfiguration() throws Exception { verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request, managementController); } - @Test - public void testCreateResource_BlueprintFactoryThrowsException() throws Exception - { + @Test(expected = IllegalArgumentException.class) + public void testCreateResource_BlueprintFactoryThrowsException() throws Exception { Request request = createMock(Request.class); Set> setProperties = getBlueprintTestProperties(); @@ -352,13 +338,7 @@ public void testCreateResource_BlueprintFactoryThrowsException() throws Exceptio replay(dao, entity, metaInfo, blueprintFactory, securityFactory, blueprint, blueprintValidator, request); // end expectations - try { - provider.createResources(request); - fail("Exception expected"); - } catch (IllegalArgumentException e) { - // expected - } - verify(dao, entity, blueprintFactory, blueprint, blueprintValidator, metaInfo, request); + provider.createResources(request); } @Test @@ -369,15 +349,14 @@ public void testCreateResources_withSecurityConfiguration() throws Exception { Set> setProperties = getBlueprintTestProperties(); Map requestInfoProperties = getTestRequestInfoProperties(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null); // set expectations expect(securityFactory.createSecurityConfigurationFromRequest(anyObject(), anyBoolean())).andReturn (securityConfiguration).once(); expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), securityConfiguration)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -495,15 +474,14 @@ public void testCreateResources_withEmptyConfiguration() throws Exception { setConfigurationProperties(setProperties); AmbariManagementController managementController = createMock(AmbariManagementController.class); Map requestInfoProperties = new HashMap<>(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); requestInfoProperties.put(Request.REQUEST_INFO_BODY_PROPERTY, "{\"configurations\":[]}"); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); @@ -542,15 +520,14 @@ public void testCreateResources_withSingleConfigurationType() throws Exception { setConfigurationProperties(setProperties); AmbariManagementController managementController = createMock(AmbariManagementController.class); Map requestInfoProperties = new HashMap<>(); - Map>> settingProperties = getSettingProperties(); + Map>> settingProperties = getSettingProperties(); requestInfoProperties.put(Request.REQUEST_INFO_BODY_PROPERTY, "{\"configurations\":[{\"configuration-type\":{\"properties\":{\"property\":\"value\"}}}]}"); Request request = createMock(Request.class); Setting setting = createStrictMock(Setting.class); // set expectations expect(blueprintFactory.createBlueprint(setProperties.iterator().next(), null)).andReturn(blueprint).once(); - blueprintValidator.validateRequiredProperties(blueprint); - blueprintValidator.validateTopology(blueprint); + blueprintValidator.validate(blueprint); expect(blueprint.getSetting()).andReturn(setting).anyTimes(); expect(setting.getProperties()).andReturn(settingProperties).anyTimes(); expect(blueprint.toEntity()).andReturn(entity); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java index 65d89c11d2f..0ab9dfb612c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java @@ -44,6 +44,7 @@ import org.apache.ambari.server.api.util.TreeNodeImpl; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.HostGroup; @@ -77,7 +78,7 @@ public void setupTest() throws Exception { AmbariMetaInfo metainfo = createNiceMock(AmbariMetaInfo.class); expect(controller.getAmbariMetaInfo()).andReturn(metainfo).anyTimes(); StackInfo stackInfo = createNiceMock(StackInfo.class); - expect(metainfo.getStack("TEST", "1.0")).andReturn(stackInfo); + expect(metainfo.getStack(new StackId("TEST", "1.0"))).andReturn(stackInfo); expect(stackInfo.getServices()).andReturn(Collections.emptySet()).anyTimes(); expect(stackInfo.getProperties()).andReturn(Collections.emptyList()).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java index d8cb701da5c..440b2e81275 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/RequestResourceProviderTest.java @@ -85,6 +85,7 @@ import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.HostRequest; import org.apache.ambari.server.topology.LogicalRequest; +import org.apache.ambari.server.topology.Setting; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.topology.TopologyRequest; import org.apache.ambari.server.utils.SecretReference; @@ -1684,7 +1685,9 @@ public void testGetLogicalRequestStatusWithNoTasks() throws Exception { expect(topologyRequest.getHostGroupInfo()).andReturn(hostGroupInfoMap).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.shouldSkipFailure()).andReturn(true).anyTimes(); + Setting setting = createNiceMock(Setting.class); + expect(blueprint.getSetting()).andReturn(setting).anyTimes(); + expect(setting.shouldSkipFailure()).andReturn(true).anyTimes(); expect(logicalRequest.getHostRequests()).andReturn(hostRequests).anyTimes(); expect(logicalRequest.constructNewPersistenceEntity()).andReturn(requestMock).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java index a7d0d479d2e..a08691862af 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java @@ -106,7 +106,7 @@ private void stackUnitIs(String name, String unit) { private String updateUnit(String serviceName, String configType, String propName, String propValue) throws InvalidTopologyException, ConfigurationTopologyException { UnitUpdater updater = new UnitUpdater(serviceName, configType); expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(serviceName, configType)).andReturn(stackConfigWithMetadata).anyTimes(); replayAll(); return updater.updateForClusterCreate(propName, propValue, Collections.emptyMap(), clusterTopology); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 895562d7756..db8c9100e06 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.capture; @@ -35,7 +36,6 @@ import static org.junit.Assert.fail; import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -45,6 +45,7 @@ import java.util.Set; import java.util.stream.Stream; +import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.ClusterRequest; import org.apache.ambari.server.controller.ConfigGroupRequest; @@ -108,6 +109,7 @@ public class AmbariContextTest { private static final AmbariContext context = new AmbariContext(); private static final AmbariManagementController controller = createNiceMock(AmbariManagementController.class); + private static final AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class); private static final ClusterController clusterController = createStrictMock(ClusterController.class); private static final HostResourceProvider hostResourceProvider = createStrictMock(HostResourceProvider.class); private static final ServiceGroupResourceProvider serviceGroupResourceProvider = createStrictMock(ServiceGroupResourceProvider.class); @@ -137,10 +139,11 @@ public class AmbariContextTest { private static final Collection group1Hosts = Arrays.asList(HOST1, HOST2); private Capture> configGroupRequestCapture = EasyMock.newCapture(); + private Setting setting = createNiceMock(Setting.class); @Before public void setUp() throws Exception { - reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, + reset(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); @@ -239,18 +242,21 @@ public void setUp() throws Exception { expect(topology.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getBlueprintName()).andReturn(BP_NAME).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.singletonMap(HOST_GROUP_1, group1Info)).anyTimes(); expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); - expect(blueprint.getServices()).andReturn(blueprintServices).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); - expect(blueprint.getStackIdsForService("service1")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.getStackIdsForService("service2")).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); - expect(blueprint.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStackIds()).andReturn(Collections.singleton(STACK_ID)).anyTimes(); + expect(topology.getServices()).andReturn(blueprintServices).anyTimes(); + expect(topology.getComponents()).andAnswer(() -> Stream.of( + ResolvedComponent.builder(new Component("s1Component1")).stackId(STACK_ID).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("s1Component2")).stackId(STACK_ID).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("s2Component1")).stackId(STACK_ID).serviceType("service2").buildPartial() + )).anyTimes(); + expect(topology.getConfiguration()).andReturn(bpConfiguration).anyTimes(); + expect(topology.getSetting()).andReturn(setting).anyTimes(); + expect(setting.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); @@ -261,6 +267,8 @@ public void setUp() throws Exception { expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(controller.getConfigHelper()).andReturn(configHelper).anyTimes(); + expect(controller.getAmbariMetaInfo()).andReturn(metaInfo).anyTimes(); + expect(metaInfo.getClusterProperties()).andReturn(emptySet()).anyTimes(); expect(clusters.getCluster(CLUSTER_NAME)).andReturn(cluster).anyTimes(); expect(clusters.getClusterById(CLUSTER_ID)).andReturn(cluster).anyTimes(); @@ -286,14 +294,14 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, + verify(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } private void replayAll() { - replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, - hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, stack, clusters, + replay(controller, clusterController, hostResourceProvider, serviceGroupResourceProvider, serviceResourceProvider, componentResourceProvider, metaInfo, + hostComponentResourceProvider, configGroupResourceProvider, topology, blueprint, setting, stack, clusters, cluster, group1Info, configHelper, configGroup1, configGroup2, host1, host2, configFactory); } @@ -336,7 +344,7 @@ public void testCreateAmbariResources() throws Exception { assertEquals(String.format("%s-%s", STACK_NAME, STACK_VERSION), clusterRequest.getStackVersion()); Set serviceGroupRequests = serviceGroupRequestCapture.getValue(); - Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), AmbariContext.DEFAULT_SERVICE_GROUP_NAME, clusterRequest.getStackVersion())); + Set expectedServiceGroupRequests = Collections.singleton(new ServiceGroupRequest(cluster.getClusterName(), STACK_NAME, clusterRequest.getStackVersion())); assertEquals(expectedServiceGroupRequests, serviceGroupRequests); Collection serviceRequests = serviceRequestCapture.getValue(); @@ -398,8 +406,6 @@ public void testCreateAmbariHostResources() throws Exception { hostResourceProvider.createHosts(anyObject(Request.class)); expectLastCall().once(); - expect(cluster.getService("service1")).andReturn(mockService1).times(2); - expect(cluster.getService("service2")).andReturn(mockService1).once(); Capture> requestsCapture = EasyMock.newCapture(); expect(controller.createHostComponents(capture(requestsCapture))).andReturn(null).once(); @@ -407,49 +413,16 @@ public void testCreateAmbariHostResources() throws Exception { replayAll(); // test - Map> componentsMap = new HashMap<>(); - Collection components = new ArrayList<>(); - components.add("component1"); - components.add("component2"); - componentsMap.put("service1", components); - components = new ArrayList<>(); - components.add("component3"); - componentsMap.put("service2", components); - - context.createAmbariHostResources(CLUSTER_ID, "host1", componentsMap); + Stream components = Stream.of( + ResolvedComponent.builder(new Component("component1", "mpack", "service1", null)).buildPartial(), + ResolvedComponent.builder(new Component("component2", "mpack", "service1", null)).buildPartial(), + ResolvedComponent.builder(new Component("component3", "mpack", "service2", null)).buildPartial() + ); + context.createAmbariHostResources(CLUSTER_ID, "host1", components); assertEquals(requestsCapture.getValue().size(), 3); } - @Test - public void testCreateAmbariHostResourcesWithMissingService() throws Exception { - // expectations - expect(cluster.getServices()).andReturn(clusterServices).anyTimes(); - - hostResourceProvider.createHosts(anyObject(Request.class)); - expectLastCall().once(); - expect(cluster.getService("service1")).andReturn(mockService1).times(2); - Capture> requestsCapture = EasyMock.newCapture(); - - expect(controller.createHostComponents(capture(requestsCapture))).andReturn(null).once(); - - replayAll(); - - // test - Map> componentsMap = new HashMap<>(); - Collection components = new ArrayList<>(); - components.add("component1"); - components.add("component2"); - componentsMap.put("service1", components); - components = new ArrayList<>(); - components.add("component3"); - componentsMap.put("service2", components); - - context.createAmbariHostResources(CLUSTER_ID, "host1", componentsMap); - - assertEquals(requestsCapture.getValue().size(), 2); - } - @Test public void testRegisterHostWithConfigGroup_createNewConfigGroup() throws Exception { // test specific expectations diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index 722345fc0fa..ff163d37671 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -19,70 +19,54 @@ package org.apache.ambari.server.topology; import static java.util.stream.Collectors.toSet; -import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; -import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.powermock.api.easymock.PowerMock.createStrictMock; -import static org.powermock.api.easymock.PowerMock.expectNew; import static org.powermock.api.easymock.PowerMock.replay; import static org.powermock.api.easymock.PowerMock.reset; import static org.powermock.api.easymock.PowerMock.verify; -import java.lang.reflect.Field; import java.util.Collection; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.ambari.server.ObjectNotFoundException; -import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider; import org.apache.ambari.server.controller.internal.BlueprintResourceProviderTest; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintEntity; -import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.StackId; -import org.easymock.EasyMockSupport; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.junit.runner.RunWith; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.inject.util.Providers; /** * BlueprintFactory unit tests. */ -@SuppressWarnings("unchecked") -@RunWith(PowerMockRunner.class) -@PrepareForTest(BlueprintImpl.class) public class BlueprintFactoryTest { private static final String BLUEPRINT_NAME = "test-blueprint"; - BlueprintFactory factory = new BlueprintFactory(); - Stack stack = createNiceMock(Stack.class); - BlueprintFactory testFactory = new TestBlueprintFactory(stack); - BlueprintDAO dao = createStrictMock(BlueprintDAO.class); - BlueprintEntity entity = createStrictMock(BlueprintEntity.class); - BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); - + private Stack stack = createNiceMock(Stack.class); + private BlueprintDAO dao = createStrictMock(BlueprintDAO.class); + private BlueprintEntity entity = createStrictMock(BlueprintEntity.class); + private BlueprintConfigEntity configEntity = createStrictMock(BlueprintConfigEntity.class); + private BlueprintFactory testFactory = new BlueprintFactory(Providers.of(dao)); @Before public void init() throws Exception { - setPrivateField(factory, "blueprintDAO", dao); - Set stackIds = ImmutableSet.of(new StackId("stack", "0.1")); Collection services = ImmutableSet.of("test-service1", "test-service2"); Collection components = ImmutableSet.of("component1", "component2"); @@ -102,24 +86,6 @@ public void tearDown() { reset(stack, dao, entity, configEntity); } - //todo: implement -// @Test -// public void testGetBlueprint() throws Exception { -// -// Collection configs = new ArrayList(); -// configs.add(configEntity); -// -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(entity).once(); -// expect(entity.getBlueprintName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); -// expect(entity.getConfigurations()).andReturn(configs).atLeastOnce(); -// -// replay(dao, entity); -// -// Blueprint blueprint = factory.getBlueprint(BLUEPRINT_NAME); -// -// -// } - @Test public void testGetMultiInstanceBlueprint() throws Exception { // prepare @@ -135,22 +101,23 @@ public void testGetMultiInstanceBlueprint() throws Exception { Stack edwStack = createNiceMock(Stack.class); expect(edwStack.getName()).andReturn(edw.getStackName()).anyTimes(); expect(edwStack.getVersion()).andReturn(edw.getStackVersion()).anyTimes(); - expectNew(Stack.class, eq(hdp.getStackName()), eq(hdp.getStackVersion()), anyObject(AmbariManagementController.class)).andReturn(hdpStack).anyTimes(); - expectNew(Stack.class, eq(edw.getStackVersion()), eq(edw.getStackVersion()), anyObject(AmbariManagementController.class)).andReturn(edwStack).anyTimes(); - replay(Stack.class, hdpStack, edwStack, dao); + replay(hdpStack, edwStack, dao); // test Blueprint blueprint = testFactory.getBlueprint(BLUEPRINT_NAME); Set mpackNames = blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(Collectors.toSet()); assertEquals(ImmutableSet.of("HDPCORE-3.0", "EDW-3.1"), mpackNames ); - MpackInstance hdpCore = - blueprint.getMpacks().stream().filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())).findAny().get(); + Optional hdpCore = blueprint.getMpacks().stream() + .filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())) + .findAny(); + assertTrue(hdpCore.isPresent()); + MpackInstance mpackInstance = hdpCore.get(); Set serviceInstanceNames = - hdpCore.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet()); + mpackInstance.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet()); assertEquals(ImmutableSet.of("ZK1", "ZK2"), serviceInstanceNames); Set serviceInstanceTypes = - hdpCore.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet()); + mpackInstance.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet()); assertEquals(ImmutableSet.of("ZOOKEEPER"), serviceInstanceTypes); Set stackIds = blueprint.getStackIds(); assertEquals(ImmutableSet.of(hdp, edw), stackIds); @@ -162,6 +129,7 @@ public void testGetBlueprint_NotFound() throws Exception { expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null).once(); replay(dao, entity, configEntity); + BlueprintFactory factory = new BlueprintFactory(Providers.of(dao)); assertNull(factory.getBlueprint(BLUEPRINT_NAME)); } @@ -183,11 +151,6 @@ public void testCreateBlueprint() throws Exception { assertEquals(2, components.size()); assertTrue(components.contains("component1")); assertTrue(components.contains("component2")); - Collection services = group1.getServices(); - assertEquals(2, services.size()); - assertTrue(services.contains("test-service1")); - assertTrue(services.contains("test-service2")); - assertTrue(group1.containsMasterComponent()); //todo: add configurations/attributes to properties Configuration configuration = group1.getConfiguration(); assertTrue(configuration.getProperties().isEmpty()); @@ -199,10 +162,6 @@ public void testCreateBlueprint() throws Exception { components = group2.getComponentNames(); assertEquals(1, components.size()); assertTrue(components.contains("component1")); - services = group2.getServices(); - assertEquals(1, services.size()); - assertTrue(services.contains("test-service1")); - assertTrue(group2.containsMasterComponent()); //todo: add configurations/attributes to properties //todo: test both v1 and v2 config syntax configuration = group2.getConfiguration(); @@ -251,24 +210,6 @@ public Blueprint createMultiInstanceBlueprint() throws Exception { return blueprint; } - @Test(expected=NoSuchStackException.class) - public void testCreateInvalidStack() throws Exception { - EasyMockSupport mockSupport = new EasyMockSupport(); - BlueprintFactory.StackFactory mockStackFactory = - mockSupport.createMock(BlueprintFactory.StackFactory.class); - - // setup mock to throw exception, to simulate invalid stack request - expect(mockStackFactory.createStack(new StackId(), null)).andThrow(new ObjectNotFoundException("Invalid Stack")); - - mockSupport.replayAll(); - - BlueprintFactory factoryUnderTest = - new BlueprintFactory(mockStackFactory); - factoryUnderTest.createStack(new StackId()); - - mockSupport.verifyAll(); - } - @Test(expected=IllegalArgumentException.class) public void testCreate_NoBlueprintName() throws Exception { Map props = BlueprintResourceProviderTest.getBlueprintTestProperties().iterator().next(); @@ -309,17 +250,6 @@ public void testCreate_HostGroupWithNoComponents() throws Exception { testFactory.createBlueprint(props, null); } - @Test(expected=IllegalArgumentException.class) - public void testCreate_HostGroupWithInvalidComponent() throws Exception { - Map props = BlueprintResourceProviderTest.getBlueprintTestProperties().iterator().next(); - // change a component name to an invalid name - ((Set>) ((Set>) props.get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). - iterator().next().get(BlueprintResourceProvider.COMPONENT_PROPERTY_ID)).iterator().next().put("name", "INVALID_COMPONENT"); - - replay(stack, dao, entity, configEntity); - testFactory.createBlueprint(props, null); - } - @Test(expected = IllegalArgumentException.class) // THEN public void verifyDefinitionsDisjointShouldRejectDuplication() { // GIVEN @@ -361,24 +291,4 @@ public void verifyStackDefinitionsAreDisjointShouldAllowDisjointStacks() { // no exception expected } - private class TestBlueprintFactory extends BlueprintFactory { - private Stack stack; - - public TestBlueprintFactory(Stack stack) { - this.stack = stack; - } - - @Override - protected Stack createStack(StackId stackId) throws NoSuchStackException { - return stack; - } - } - - - private void setPrivateField(Object o, String field, Object value) throws Exception { - Class c = o.getClass(); - Field f = c.getDeclaredField(field); - f.setAccessible(true); - f.set(o, value); - } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index 362f4e4889f..19827e70432 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -23,14 +23,11 @@ import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.reset; import static org.easymock.EasyMock.verify; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -40,6 +37,7 @@ import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.state.SecurityType; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; @@ -48,11 +46,9 @@ */ public class BlueprintImplTest { private static final Map>> EMPTY_ATTRIBUTES = new HashMap<>(); - private static final Map> EMPTY_PROPERTIES = new HashMap<>(); - private static final Configuration EMPTY_CONFIGURATION = new Configuration(EMPTY_PROPERTIES, EMPTY_ATTRIBUTES); + private static final Configuration EMPTY_CONFIGURATION = Configuration.createEmpty(); Stack stack = createNiceMock(Stack.class); - Setting setting = createNiceMock(Setting.class); HostGroup group1 = createMock(HostGroup.class); HostGroup group2 = createMock(HostGroup.class); Set hostGroups = new HashSet<>(); @@ -62,10 +58,9 @@ public class BlueprintImplTest { Map hdfsProps = new HashMap<>(); Configuration configuration = new Configuration(properties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); private final org.apache.ambari.server.configuration.Configuration serverConfig = createNiceMock(org.apache.ambari.server.configuration.Configuration.class); - private final BlueprintValidator blueprintValidator = new BlueprintValidatorImpl(serverConfig); @Before - public void setup() throws NoSuchFieldException, IllegalAccessException { + public void setup() { properties.put("hdfs-site", hdfsProps); hdfsProps.put("foo", "val"); hdfsProps.put("bar", "val"); @@ -106,12 +101,11 @@ public void setup() throws NoSuchFieldException, IllegalAccessException { requiredService2Properties.add(new Stack.ConfigProperty("category2", "prop2", null)); expect(stack.getRequiredConfigurationProperties("HDFS")).andReturn(requiredHDFSProperties).anyTimes(); expect(stack.getRequiredConfigurationProperties("SERVICE2")).andReturn(requiredService2Properties).anyTimes(); - - setupConfigurationWithGPLLicense(true); } @Test public void testValidateConfigurations__basic_positive() throws Exception { + // GIVEN expect(group1.getCardinality()).andReturn("1").atLeastOnce(); expect(group1.getComponents()).andReturn(Arrays.asList(new Component("c1"), new Component("c2"))).atLeastOnce(); expect(group2.getCardinality()).andReturn("1").atLeastOnce(); @@ -125,10 +119,12 @@ public void testValidateConfigurations__basic_positive() throws Exception { category2Props.put("prop2", "val"); SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, securityConfiguration, null); - blueprintValidator.validateRequiredProperties(blueprint); + + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, securityConfiguration, null); BlueprintEntity entity = blueprint.toEntity(); + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.KERBEROS); assertTrue(entity.getSecurityDescriptorReference().equals("testRef")); @@ -136,6 +132,7 @@ public void testValidateConfigurations__basic_positive() throws Exception { @Test public void testValidateConfigurations__hostGroupConfig() throws Exception { + // GIVEN Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); group2Props.put("category2", group2Category2Props); @@ -163,13 +160,18 @@ public void testValidateConfigurations__hostGroupConfig() throws Exception { hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); + + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); BlueprintEntity entity = blueprint.toEntity(); + + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.NONE); assertTrue(entity.getSecurityDescriptorReference() == null); } + + @Ignore // move out NAMENODE-specific test @Test public void testValidateConfigurations__hostGroupConfigForNameNodeHAPositive() throws Exception { Map> group2Props = new HashMap<>(); @@ -203,16 +205,18 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAPositive() t hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); + // WHEN + Blueprint blueprint = new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); BlueprintEntity entity = blueprint.toEntity(); + // THEN verify(stack, group1, group2, serverConfig); assertTrue(entity.getSecurityType() == SecurityType.NONE); assertTrue(entity.getSecurityDescriptorReference() == null); } - @Test(expected= IllegalArgumentException.class) + @Ignore // move out NAMENODE-specific test + @Test(expected = IllegalArgumentException.class) public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHostGroups() throws Exception { Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); @@ -244,11 +248,13 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHos hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group3%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); + + // WHEN + new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); } - @Test(expected= IllegalArgumentException.class) + + @Ignore // move out NAMENODE-specific test + @Test(expected = IllegalArgumentException.class) public void testValidateConfigurations__hostGroupConfigForNameNodeHAMappedSameHostGroup() throws Exception { Map> group2Props = new HashMap<>(); Map group2Category2Props = new HashMap<>(); @@ -280,695 +286,9 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAMappedSameHo hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - @Test(expected = InvalidTopologyException.class) - public void testValidateConfigurations__secretReference() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> group2Props = new HashMap<>(); - Map group2Category2Props = new HashMap<>(); - group2Props.put("category2", group2Category2Props); - group2Category2Props.put("prop2", "val"); - hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test(expected = GPLLicenseNotAcceptedException.class) - public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - setupConfigurationWithGPLLicense(false); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); + // WHEN + new BlueprintImpl("test", hostGroups, emptySet(), emptySet(), configuration, null, null); } - @Test(expected = GPLLicenseNotAcceptedException.class) - public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - setupConfigurationWithGPLLicense(false); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test - public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyException, - GPLLicenseNotAcceptedException, NoSuchFieldException, IllegalAccessException { - Map> lzoProperties = new HashMap<>(); - lzoProperties.put("core-site", new HashMap(){{ - put(BlueprintValidatorImpl.LZO_CODEC_CLASS_PROPERTY_NAME, BlueprintValidatorImpl.LZO_CODEC_CLASS); - put(BlueprintValidatorImpl.CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + BlueprintValidatorImpl.LZO_CODEC_CLASS); - }}); - Configuration lzoUsageConfiguration = new Configuration(lzoProperties, EMPTY_ATTRIBUTES, EMPTY_CONFIGURATION); - - expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); - replay(stack, group1, group2, serverConfig); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), lzoUsageConfiguration, null, null); - blueprintValidator.validateRequiredProperties(blueprint); - verify(stack, group1, group2, serverConfig); - } - - @Test - public void testAutoSkipFailureEnabled() { - HashMap skipFailureSetting = new HashMap<>(); - skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); - expect(stack.getName()).andReturn("HDPCORE").anyTimes(); - expect(stack.getVersion()).andReturn("3.0.0.0").anyTimes(); - expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); - replay(stack, setting); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, setting); - assertTrue(blueprint.shouldSkipFailure()); - - verify(stack, setting); - } - - @Test - public void testAutoSkipFailureDisabled() { - HashMap skipFailureSetting = new HashMap<>(); - skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "false"); - expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); - replay(stack, setting); - - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, emptySet(), emptySet(), configuration, null, setting); - assertFalse(blueprint.shouldSkipFailure()); - - verify(stack, setting); - } - - private org.apache.ambari.server.configuration.Configuration setupConfigurationWithGPLLicense(boolean isGPLAllowed) { - reset(serverConfig); - expect(serverConfig.getGplLicenseAccepted()).andReturn(isGPLAllowed).atLeastOnce(); - return serverConfig; - } - - //todo: ensure coverage for these existing tests - - // private void validateEntity(BlueprintEntity entity, boolean containsConfig) { -// assertEquals(BLUEPRINT_NAME, entity.getBlueprintName()); -// -// StackEntity stackEntity = entity.getStack(); -// assertEquals("test-stack-name", stackEntity.getStackName()); -// assertEquals("test-stack-version", stackEntity.getStackVersion()); -// -// Collection hostGroupEntities = entity.getHostGroups(); -// -// assertEquals(2, hostGroupEntities.size()); -// for (HostGroupEntity hostGroup : hostGroupEntities) { -// assertEquals(BLUEPRINT_NAME, hostGroup.getBlueprintName()); -// assertNotNull(hostGroup.getBlueprintEntity()); -// Collection componentEntities = hostGroup.getComponents(); -// if (hostGroup.getName().equals("group1")) { -// assertEquals("1", hostGroup.getCardinality()); -// assertEquals(2, componentEntities.size()); -// Iterator componentIterator = componentEntities.iterator(); -// String name = componentIterator.next().getName(); -// assertTrue(name.equals("component1") || name.equals("component2")); -// String name2 = componentIterator.next().getName(); -// assertFalse(name.equals(name2)); -// assertTrue(name2.equals("component1") || name2.equals("component2")); -// } else if (hostGroup.getName().equals("group2")) { -// assertEquals("2", hostGroup.getCardinality()); -// assertEquals(1, componentEntities.size()); -// HostGroupComponentEntity componentEntity = componentEntities.iterator().next(); -// assertEquals("component1", componentEntity.getName()); -// -// if (containsConfig) { -// Collection configurations = hostGroup.getConfigurations(); -// assertEquals(1, configurations.size()); -// HostGroupConfigEntity hostGroupConfigEntity = configurations.iterator().next(); -// assertEquals(BLUEPRINT_NAME, hostGroupConfigEntity.getBlueprintName()); -// assertSame(hostGroup, hostGroupConfigEntity.getHostGroupEntity()); -// assertEquals("core-site", hostGroupConfigEntity.getType()); -// Map properties = gson.>fromJson( -// hostGroupConfigEntity.getConfigData(), Map.class); -// assertEquals(1, properties.size()); -// assertEquals("anything", properties.get("my.custom.hg.property")); -// } -// } else { -// fail("Unexpected host group name"); -// } -// } -// Collection configurations = entity.getConfigurations(); -// if (containsConfig) { -// assertEquals(1, configurations.size()); -// BlueprintConfigEntity blueprintConfigEntity = configurations.iterator().next(); -// assertEquals(BLUEPRINT_NAME, blueprintConfigEntity.getBlueprintName()); -// assertSame(entity, blueprintConfigEntity.getBlueprintEntity()); -// assertEquals("core-site", blueprintConfigEntity.getType()); -// Map properties = gson.>fromJson( -// blueprintConfigEntity.getConfigData(), Map.class); -// assertEquals(2, properties.size()); -// assertEquals("480", properties.get("fs.trash.interval")); -// assertEquals("8500", properties.get("ipc.client.idlethreshold")); -// } else { -// assertEquals(0, configurations.size()); -// } -// } - - - - // @Test -// public void testCreateResource_Validate__Cardinality__ExternalComponent() throws Exception { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// ((Set>) setProperties.iterator().next().get("configurations")). -// add(Collections.singletonMap("global/hive_database", "Existing MySQL Database")); -// -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("MYSQL_SERVER"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(blueprintFactory.createBlueprint(setProperties.iterator().next())).andReturn(blueprint).once(); -// expect(blueprintValidator.validateRequiredProperties()).andReturn(Collections.>>emptyMap()).once(blueprint); -// expect(blueprint.toEntity()).andReturn(entity); -// expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).atLeastOnce(); -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("MYSQL_SERVER").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "MYSQL_SERVER")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__MultipleDependencyInstances() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// DependencyInfo dependencyInfo = new DependencyInfo(); -// AutoDeployInfo autoDeployInfo = new AutoDeployInfo(); -// autoDeployInfo.setEnabled(false); -// dependencyInfo.setAutoDeploy(autoDeployInfo); -// dependencyInfo.setScope("cluster"); -// dependencyInfo.setName("test-service/component1"); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("component2").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component2")). -// andReturn(Collections.singletonList(dependencyInfo)).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__AutoCommit() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// // remove component2 from BP -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// DependencyInfo dependencyInfo = new DependencyInfo(); -// AutoDeployInfo autoDeployInfo = new AutoDeployInfo(); -// autoDeployInfo.setEnabled(true); -// autoDeployInfo.setCoLocate("test-service/component1"); -// dependencyInfo.setAutoDeploy(autoDeployInfo); -// dependencyInfo.setScope("cluster"); -// dependencyInfo.setName("test-service/component2"); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("component2").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component2")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.singletonList(dependencyInfo)).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__Cardinality__Fail() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException { -// -// Set> setProperties = getTestProperties(); -// setConfigurationProperties(setProperties); -// -// Iterator iter = ((HashSet>>>) setProperties.iterator().next(). -// get(BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)). -// iterator().next().get("components").iterator(); -// iter.next(); -// iter.remove(); -// -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// Capture> serviceComponentRequestCapture = EasyMock.newCapture(); -// Capture stackConfigurationRequestCapture = EasyMock.newCapture(); -// Capture stackLevelConfigurationRequestCapture = EasyMock.newCapture(); -// Request request = createMock(Request.class); -// StackServiceResponse stackServiceResponse = createMock(StackServiceResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse = createNiceMock(StackServiceComponentResponse.class); -// StackServiceComponentResponse stackServiceComponentResponse2 = createNiceMock(StackServiceComponentResponse.class); -// Set setServiceComponents = new HashSet(); -// setServiceComponents.add(stackServiceComponentResponse); -// setServiceComponents.add(stackServiceComponentResponse2); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("MYSQL_SERVER"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.singleton(stackServiceResponse)); -// expect(stackServiceResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceResponse.getExcludedConfigTypes()).andReturn(Collections.emptySet()); -// -// expect(managementController.getStackComponents(capture(serviceComponentRequestCapture))).andReturn(setServiceComponents).anyTimes(); -// expect(stackServiceComponentResponse.getCardinality()).andReturn("2").anyTimes(); -// expect(stackServiceComponentResponse.getComponentName()).andReturn("component1").anyTimes(); -// expect(stackServiceComponentResponse.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// expect(stackServiceComponentResponse2.getCardinality()).andReturn("1").anyTimes(); -// expect(stackServiceComponentResponse2.getComponentName()).andReturn("MYSQL_SERVER").anyTimes(); -// expect(stackServiceComponentResponse2.getServiceName()).andReturn("test-service").anyTimes(); -// expect(stackServiceComponentResponse2.getStackName()).andReturn("test-stack-name").anyTimes(); -// expect(stackServiceComponentResponse2.getStackVersion()).andReturn("test-stack-version").anyTimes(); -// -// expect(managementController.getStackConfigurations(Collections.singleton(capture(stackConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// expect(managementController.getStackLevelConfigurations(Collections.singleton(capture(stackLevelConfigurationRequestCapture)))). -// andReturn(Collections.emptySet()); -// -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "MYSQL_SERVER")). -// andReturn(Collections.emptyList()).anyTimes(); -// expect(metaInfo.getComponentDependencies("test-stack-name", "test-stack-version", "test-service", "component1")). -// andReturn(Collections.emptyList()).anyTimes(); -// -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component2")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// -// replay(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// try { -// provider.createResources(request); -// fail("Expected validation failure for MYSQL_SERVER"); -// } catch (IllegalArgumentException e) { -// // expected -// } -// -// verify(dao, metaInfo, request, managementController, stackServiceResponse, -// stackServiceComponentResponse, stackServiceComponentResponse2); -// } - -// @Test -// public void testCreateResource_Validate__AmbariServerComponent() throws AmbariException, ResourceAlreadyExistsException, -// SystemException, UnsupportedPropertyException, NoSuchParentResourceException -// { -// Request request = createMock(Request.class); -// AmbariManagementController managementController = createMock(AmbariManagementController.class); -// Capture> stackServiceRequestCapture = EasyMock.newCapture(); -// -// Map services = new HashMap(); -// ServiceInfo service = new ServiceInfo(); -// service.setName("test-service"); -// services.put("test-service", service); -// -// List serviceComponents = new ArrayList(); -// ComponentInfo component1 = new ComponentInfo(); -// component1.setName("component1"); -// ComponentInfo component2 = new ComponentInfo(); -// component2.setName("component2"); -// serviceComponents.add(component1); -// serviceComponents.add(component2); -// -// -// Set> setProperties = getTestProperties(); -// ((HashSet>) ((HashSet>) setProperties.iterator().next().get( -// BlueprintResourceProvider.HOST_GROUP_PROPERTY_ID)).iterator().next().get("components")). -// iterator().next().put("name", "AMBARI_SERVER"); -// -// Capture entityCapture = EasyMock.newCapture(); -// -// // set expectations -// expect(managementController.getStackServices(capture(stackServiceRequestCapture))).andReturn( -// Collections.emptySet()); -// expect(request.getProperties()).andReturn(setProperties); -// expect(request.getRequestInfoProperties()).andReturn(Collections.emptyMap()); -// expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null); -// expect(metaInfo.getServices("test-stack-name", "test-stack-version")).andReturn(services).anyTimes(); -// expect(metaInfo.getComponentsByService("test-stack-name", "test-stack-version", "test-service")). -// andReturn(serviceComponents).anyTimes(); -// expect(metaInfo.getComponentToService("test-stack-name", "test-stack-version", "component1")). -// andReturn("test-service").anyTimes(); -// expect(metaInfo.getService("test-stack-name", "test-stack-version", "test-service")).andReturn(service).anyTimes(); -// -// dao.create(capture(entityCapture)); -// -// replay(dao, metaInfo, request, managementController); -// // end expectations -// -// ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( -// Resource.Type.Blueprint, -// PropertyHelper.getPropertyIds(Resource.Type.Blueprint), -// PropertyHelper.getKeyPropertyIds(Resource.Type.Blueprint), -// managementController); -// -// AbstractResourceProviderTest.TestObserver observer = new AbstractResourceProviderTest.TestObserver(); -// ((ObservableResourceProvider)provider).addObserver(observer); -// -// provider.createResources(request); -// -// ResourceProviderEvent lastEvent = observer.getLastEvent(); -// assertNotNull(lastEvent); -// assertEquals(Resource.Type.Blueprint, lastEvent.getResourceType()); -// assertEquals(ResourceProviderEvent.Type.Create, lastEvent.getType()); -// assertEquals(request, lastEvent.getRequest()); -// assertNull(lastEvent.getPredicate()); -// -// verify(dao, metaInfo, request, managementController); -// } - - } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index c4ba8d8ca6c..7d51906a356 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -31,13 +31,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorBlueprintProcessor; @@ -236,8 +235,8 @@ private Capture> testProcessWithKerberos(String blueprintP expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("testConfigType")).andReturn("KERBEROS").anyTimes(); @@ -246,28 +245,20 @@ private Capture> testProcessWithKerberos(String blueprintP expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); - Set services = new HashSet<>(); - services.add("HDFS"); - services.add("KERBEROS"); - services.add("ZOOKEPER"); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + Set services = ImmutableSet.of("HDFS", "KERBEROS", "ZOOKEEPER"); + expect(topology.getServices()).andReturn(services).anyTimes(); expect(stack.getConfiguration(services)).andReturn(stackDefaultConfig).once(); - List hdfsComponents = new ArrayList<>(); - hdfsComponents.add("NAMENODE"); - List kerberosComponents = new ArrayList<>(); - kerberosComponents.add("KERBEROS_CLIENT"); - List zookeeperComponents = new ArrayList<>(); - zookeeperComponents.add("ZOOKEEPER_SERVER"); - - expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(topology.getComponents()).andAnswer(() -> Stream.of( + ResolvedComponent.builder(new Component("NAMENODE")).serviceType("HDFS").buildPartial(), + ResolvedComponent.builder(new Component("KERBEROS")).serviceType("KERBEROS_CLIENT").buildPartial(), + ResolvedComponent.builder(new Component("ZOOKEEPER_SERVER")).serviceType("ZOOKEEPER").buildPartial() + )).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); - expect(blueprint.isValidConfigType("testConfigType")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("testConfigType")).andReturn(true).anyTimes(); expect(topology.getConfiguration()).andReturn(blueprintConfig).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); @@ -316,7 +307,7 @@ private Capture> testProcessWithKerberos(String blueprintP } @Test - public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Exception { + public void testProcessClusterConfigRequestDontIncludeKerberosConfigs() throws Exception { Map> existingConfig = new HashMap<>(); Configuration stackConfig = new Configuration(existingConfig, @@ -329,40 +320,25 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex expect(controller.getClusters()).andReturn(clusters).anyTimes(); expect(clusters.getCluster("testCluster")).andReturn(cluster).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getAllConfigurationTypes(anyString())).andReturn(Collections.singletonList("testConfigType")).anyTimes(); expect(stack.getExcludedConfigurationTypes(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(anyString(), anyString())).andReturn(Collections.emptyMap()).anyTimes(); - Set services = new HashSet<>(); - services.add("HDFS"); - services.add("KERBEROS"); - services.add("ZOOKEPER"); - expect(blueprint.getServices()).andReturn(services).anyTimes(); - - List hdfsComponents = new ArrayList<>(); - hdfsComponents.add("NAMENODE"); - List kerberosComponents = new ArrayList<>(); - kerberosComponents.add("KERBEROS_CLIENT"); - List zookeeperComponents = new ArrayList<>(); - zookeeperComponents.add("ZOOKEEPER_SERVER"); - - expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); - + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "KERBEROS", "ZOOKEEPER")).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); + expect(topology.getComponents()).andAnswer(Stream::empty).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); - expect(topology.getClusterId()).andReturn(Long.valueOf(1)).anyTimes(); + expect(topology.getClusterId()).andReturn(1L).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); - expect(ambariContext.getClusterName(Long.valueOf(1))).andReturn("testCluster").anyTimes(); + expect(ambariContext.getClusterName(1L)).andReturn("testCluster").anyTimes(); expect(ambariContext.createConfigurationRequests(EasyMock.anyObject())).andReturn(Collections .emptyList()).anyTimes(); @@ -396,14 +372,14 @@ public void testProcessClusterConfigRequestRemoveUnusedConfigTypes() throws Exce expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); - expect(blueprint.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("admin-properties")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("yarn-site")).andReturn(false).anyTimes(); - expect(blueprint.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("global")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("admin-properties")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("yarn-site")).andReturn(false).anyTimes(); + expect(topology.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("global")).andReturn(true).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); @@ -446,12 +422,12 @@ public void testProcessClusterConfigRequestWithOnlyHostGroupConfigRemoveUnusedCo expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(topology.getHostGroupInfo()).andReturn(hostGroupInfoMap); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); - expect(blueprint.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); - expect(blueprint.isValidConfigType("global")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("hdfs-site")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("cluster-env")).andReturn(true).anyTimes(); + expect(topology.isValidConfigType("global")).andReturn(true).anyTimes(); expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index 55db7684f18..dab906242da 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -17,6 +17,7 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; @@ -26,7 +27,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import static org.junit.Assert.assertEquals; @@ -64,6 +64,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; @@ -80,6 +81,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -91,6 +93,7 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -98,6 +101,9 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; @@ -166,6 +172,8 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); @@ -187,8 +195,10 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -197,7 +207,6 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { private List topologyValidators = new ArrayList<>(); - private Capture clusterTopologyCapture; private Capture> configRequestPropertiesCapture; private Capture> configRequestPropertiesCapture2; private Capture> configRequestPropertiesCapture3; @@ -206,7 +215,10 @@ public class ClusterDeployWithStartOnlyTest extends EasyMockSupport { @Before public void setup() throws Exception { - clusterTopologyCapture = newCapture(); + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); configRequestPropertiesCapture3 = newCapture(); @@ -244,22 +256,18 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -289,8 +297,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -302,30 +315,33 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(ProvisionAction.START_ONLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( @@ -345,7 +361,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index bac0e5b9f56..13d5b175a0c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -28,7 +29,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import java.lang.reflect.Field; @@ -65,6 +65,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; @@ -81,6 +82,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -92,6 +94,7 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -99,6 +102,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; @@ -163,6 +169,9 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -183,8 +192,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -193,7 +204,6 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private List topologyValidators = new ArrayList<>(); - private Capture clusterTopologyCapture; private Capture> configRequestPropertiesCapture; private Capture> configRequestPropertiesCapture2; private Capture> configRequestPropertiesCapture3; @@ -203,7 +213,10 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp @Before public void setup() throws Exception { - clusterTopologyCapture = newCapture(); + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); configRequestPropertiesCapture3 = newCapture(); @@ -241,22 +254,18 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -286,8 +295,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -299,36 +313,39 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(INSTALL_AND_START).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Arrays.asList("component1")) .anyTimes(); expect(group1.getComponentNames(ProvisionAction.START_ONLY)).andReturn(Collections.emptyList()) .anyTimes(); - expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(ProvisionAction.INSTALL_ONLY)).andReturn(Collections.emptyList()).anyTimes(); expect(group2.getComponentNames(ProvisionAction.START_ONLY)).andReturn(Collections.emptyList()) .anyTimes(); - expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( @@ -348,7 +365,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), (String) eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index 3e11f5e8c64..d0841abdc17 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_ONLY; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -28,7 +29,6 @@ import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.easymock.EasyMock.isA; -import static org.easymock.EasyMock.isNull; import static org.easymock.EasyMock.newCapture; import java.lang.reflect.Field; @@ -65,6 +65,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMockRule; @@ -81,6 +82,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @RunWith(PowerMockRunner.class) @@ -93,6 +95,7 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private static final String STACK_NAME = "test-stack"; private static final String STACK_VERSION = "test-stack-version"; private static final StackId STACK_ID = new StackId(STACK_NAME, STACK_VERSION); + private static final Setting SETTING = new Setting(Collections.emptyMap()); @Rule public EasyMockRule mocks = new EasyMockRule(this); @@ -100,6 +103,9 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @TestSubject private TopologyManager topologyManager = new TopologyManager(); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @Mock(type = MockType.NICE) private Blueprint blueprint; @@ -165,6 +171,9 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Mock private TopologyValidatorService topologyValidatorServiceMock; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -185,8 +194,10 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList("component1", "component2", "component3"); - private Collection group2Components = Arrays.asList("component3", "component4"); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -204,6 +215,10 @@ public class ClusterInstallWithoutStartTest extends EasyMockSupport { @Before public void setup() throws Exception { + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + clusterTopologyCapture = newCapture(); configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); @@ -242,26 +257,26 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getSetting()).andReturn(SETTING).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.isValidConfigType(anyString())).andReturn(true).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() - expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); - expect(stack.getAllConfigurationTypes("service2")).andReturn(Arrays.asList("service2-site", "service2-env")).anyTimes(); + List configTypes1 = Arrays.asList("service1-site", "service1-env"); + List configTypes2 = Arrays.asList("service2-site", "service2-env"); + expect(stack.getConfigurationTypes("service1")).andReturn(configTypes1).anyTimes(); + expect(stack.getConfigurationTypes("service2")).andReturn(configTypes2).anyTimes(); + expect(stack.getAllConfigurationTypes("service1")).andReturn(configTypes1).anyTimes(); + expect(stack.getAllConfigurationTypes("service2")).andReturn(configTypes2).anyTimes(); expect(stack.getAutoDeployInfo("component1")).andReturn(null).anyTimes(); expect(stack.getAutoDeployInfo("component2")).andReturn(null).anyTimes(); expect(stack.getAutoDeployInfo("component3")).andReturn(null).anyTimes(); @@ -287,8 +302,13 @@ public void setup() throws Exception { expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -300,31 +320,34 @@ public void setup() throws Exception { expect(request.getProvisionAction()).andReturn(INSTALL_ONLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); expect(request.getRepositoryVersion()).andReturn("1").anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of( + "group1", ImmutableSet.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial() + ), + "group2", ImmutableSet.of( + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + ) + )).anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(group1.getComponents()).andReturn(group1Components).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(group2.getComponents()).andReturn(group2Components).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); // Create partial mock to allow actual logical request creation logicalRequestFactory = createMockBuilder(LogicalRequestFactory.class).addMockedMethod( @@ -344,7 +367,7 @@ public void setup() throws Exception { expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), eq("1"), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), eq("1"), anyLong()); expectLastCall().once(); expect(ambariContext.getNextRequestId()).andReturn(1L).once(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java index e51ce5f63e1..5e81730e28d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java @@ -33,6 +33,7 @@ import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; /** @@ -49,6 +50,7 @@ public class ClusterTopologyImplTest { private static final HostGroup group2 = createNiceMock(HostGroup.class); private static final HostGroup group3 = createNiceMock(HostGroup.class); private static final HostGroup group4 = createNiceMock(HostGroup.class); + private final AmbariContext ambariContext = createNiceMock(AmbariContext.class); private final Map hostGroupInfoMap = new HashMap<>(); private final Map hostGroupMap = new HashMap<>(); @@ -163,7 +165,7 @@ public void testCreate_duplicateHosts() throws Exception { replayAll(); // should throw exception due to duplicate host - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } @Test @@ -172,9 +174,10 @@ public void test_GetHostAssigmentForComponents() throws Exception { replayAll(); - new ClusterTopologyImpl(null, request).getHostAssignmentsForComponent("component1"); + new ClusterTopologyImpl(ambariContext, request).getHostAssignmentsForComponent("component1"); } + @Ignore @Test(expected = InvalidTopologyException.class) public void testCreate_NNHAInvaid() throws Exception { bpconfiguration.setProperty("hdfs-site", "dfs.nameservices", "val"); @@ -182,10 +185,11 @@ public void testCreate_NNHAInvaid() throws Exception { hostGroupInfoMap.get("group4").removeHost("host5"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); hostGroupInfoMap.get("group4").addHost("host5"); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForStandby() throws Exception { expect(group4.getName()).andReturn("group4"); @@ -194,9 +198,10 @@ public void testCreate_NNHAHostNameNotCorrectForStandby() throws Exception { bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "val"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForActive() throws Exception { expect(group4.getName()).andReturn("group4"); @@ -205,9 +210,10 @@ public void testCreate_NNHAHostNameNotCorrectForActive() throws Exception { bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "host5"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } + @Ignore @Test(expected = IllegalArgumentException.class) public void testCreate_NNHAHostNameNotCorrectForStandbyWithActiveAsVariable() throws Exception { expect(group4.getName()).andReturn("group4"); @@ -216,7 +222,7 @@ public void testCreate_NNHAHostNameNotCorrectForStandbyWithActiveAsVariable() th bpconfiguration.setProperty("hadoop-env", "dfs_ha_initial_namenode_standby", "host6"); TestTopologyRequest request = new TestTopologyRequest(TopologyRequest.Type.PROVISION); replayAll(); - new ClusterTopologyImpl(null, request); + new ClusterTopologyImpl(ambariContext, request); } private class TestTopologyRequest implements TopologyRequest { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java index a5265f67883..b0c3958d322 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java @@ -97,6 +97,9 @@ public class LogicalRequestTest extends EasyMockSupport { @Mock private HostGroup hostGroup2; + @Mock + private Setting setting; + @Before public void setup() throws Exception { @@ -113,8 +116,10 @@ public void setup() throws Exception { expect(clusterTopology.getClusterId()).andReturn(clusterId).anyTimes(); expect(clusterTopology.getProvisionAction()).andReturn(ProvisionAction.INSTALL_ONLY).anyTimes(); expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getBlueprintName()).andReturn("blueprintDef").anyTimes(); + expect(clusterTopology.getSetting()).andReturn(setting).anyTimes(); expect(blueprint.getName()).andReturn("blueprintDef").anyTimes(); - expect(blueprint.shouldSkipFailure()).andReturn(true).anyTimes(); + expect(setting.shouldSkipFailure()).andReturn(true).anyTimes(); PowerMock.reset(AmbariServer.class); @@ -122,9 +127,6 @@ public void setup() throws Exception { expect(AmbariServer.getController()).andReturn(controller).anyTimes(); PowerMock.replay(AmbariServer.class); - - - } @Test @@ -198,7 +200,7 @@ public void testPersistedRequestsWithReservedHosts() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -301,7 +303,7 @@ public void testPersistedRequestsWithHostPredicate() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_2"))).andReturn(hostGroup2).atLeastOnce(); - expect(hostGroup2.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_2")).andReturn(false).atLeastOnce(); replayAll(); @@ -412,7 +414,7 @@ public void testRemoveHostRequestByHostName() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -510,7 +512,7 @@ public void testRemovePendingHostRequests() throws Exception { expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).atLeastOnce(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(reservedHostRequestEntities).atLeastOnce(); expect(blueprint.getHostGroup(eq("host_group_1"))).andReturn(hostGroup1).atLeastOnce(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).atLeastOnce(); + expect(clusterTopology.containsMasterComponent("host_group_1")).andReturn(false).atLeastOnce(); replayAll(); @@ -596,7 +598,7 @@ private LogicalRequest createTopologyRequestByHostCount(int hostCount, String ho expect(logicalRequestEntity.getTopologyRequestEntity()).andReturn(topologyRequestEntity).anyTimes(); expect(logicalRequestEntity.getTopologyHostRequestEntities()).andReturn(hostRequests).anyTimes(); expect(blueprint.getHostGroup(eq(hostGroupEntity.getName()))).andReturn(hostGroup1).anyTimes(); - expect(hostGroup1.containsMasterComponent()).andReturn(false).anyTimes(); + expect(clusterTopology.containsMasterComponent(hostGroupName)).andReturn(false).anyTimes(); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java index f75e6c19d95..94c412d402d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingFactoryTest.java @@ -39,13 +39,9 @@ public class SettingFactoryTest { */ @Test public void testGetSettingWithSetOfProperties() { - SettingFactory settingFactory = new SettingFactory(); - Map>> properties; - - Setting setting = settingFactory.getSetting(createSettingWithSetOfProperties()); - Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); + Setting setting = SettingFactory.getSetting(createSettingWithSetOfProperties()); + Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); assertEquals(propertyValues.size(), 1); - assertEquals(propertyValues.iterator().next().get(Setting.SETTING_NAME_RECOVERY_ENABLED), "true"); } @@ -54,13 +50,9 @@ public void testGetSettingWithSetOfProperties() { */ @Test public void testGetSettingWithoutSetOfProperties() { - SettingFactory settingFactory = new SettingFactory(); - Map>> properties; - - Setting setting = settingFactory.getSetting(createSettingWithoutSetOfProperties()); - Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); + Setting setting = SettingFactory.getSetting(createSettingWithoutSetOfProperties()); + Set> propertyValues = setting.getSettingValue(Setting.SETTING_NAME_RECOVERY_SETTINGS); assertEquals(propertyValues.size(), 1); - assertEquals(propertyValues.iterator().next().get(Setting.SETTING_NAME_RECOVERY_ENABLED), "true"); } @@ -72,13 +64,11 @@ public void testGetSettingWithoutSetOfProperties() { * } * ] * } - * - * @return */ private Collection> createSettingWithSetOfProperties() { - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); + Set> setting1 = new HashSet<>(); + Set> setting2 = new HashSet<>(); // Setting 1: Property1 HashMap setting1Properties1 = new HashMap<>(); @@ -117,8 +107,6 @@ private Collection> createSettingWithSetOfProperties() { * "recovery_enabled":"true" * } * } - * - * @return */ private Collection> createSettingWithoutSetOfProperties() { // Setting 2: Property1 and Property2 @@ -130,7 +118,7 @@ private Collection> createSettingWithoutSetOfProperties() { setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - Set> setting2 = new HashSet<>(); + Set> setting2 = new HashSet<>(); setting2.add(setting2Properties1); setting2.add(setting2Properties2); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java index 7ce3b38d13b..ecd5acbcd32 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/SettingTest.java @@ -19,72 +19,129 @@ package org.apache.ambari.server.topology; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import org.junit.BeforeClass; import org.junit.Test; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + /** * Test the Setting class */ public class SettingTest { - /** - * Test get and set of entire setting. - */ - @Test - public void testGetProperties() { - Map>> properties = new HashMap<>(); - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); - Set> setting3 = new HashSet<>(); - Set> setting4 = new HashSet<>(); + + private static Setting setting; + private static Map>> properties; + private static Set> serviceSettings; + + @BeforeClass + public static void setup() { + properties = new HashMap<>(); + Set> recoverySettings = new HashSet<>(); + Set> deploymentSettings = new HashSet<>(); + Set> repositorySettings = new HashSet<>(); // Setting 1: Property1 - HashMap setting1Properties1 = new HashMap<>(); + Map setting1Properties1 = new HashMap<>(); setting1Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true"); - setting1.add(setting1Properties1); + recoverySettings.add(setting1Properties1); // Setting 2: Property1 and Property2 - HashMap setting2Properties1 = new HashMap<>(); - setting2Properties1.put(Setting.SETTING_NAME_NAME, "HDFS"); - setting2Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - - HashMap setting2Properties2 = new HashMap<>(); - setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); - setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); - - setting2.add(setting2Properties1); - setting2.add(setting2Properties2); + Map hdfs = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "HDFS", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map yarn = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "YARN" + // no RECOVERY_ENABLED value for YARN + ); + + Map tez = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "TEZ", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + serviceSettings = ImmutableSet.of(hdfs, yarn, tez); + + Map hdfsClient = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "HDFS_CLIENT", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map namenode = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "NAMENODE", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + Map datanode = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "DATANODE" + // no RECOVERY_ENABLED value + ); + + Map yarnClient = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "YARN_CLIENT", + Setting.SETTING_NAME_RECOVERY_ENABLED, "false" + ); + + Map resourceManager = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "RESOURCE_MANAGER", + Setting.SETTING_NAME_RECOVERY_ENABLED, "true" + ); + + Map nodeManager = ImmutableMap.of( + Setting.SETTING_NAME_NAME, "NODE_MANAGER" + // no RECOVERY_ENABLED value + ); + + Set> componentSettings = ImmutableSet.>builder() + .addAll(ImmutableSet.of(hdfsClient, namenode, datanode)) + .addAll(ImmutableSet.of(yarnClient, resourceManager, nodeManager)) + .build(); //Setting 3: Property 1 - HashMap setting3Properties1 = new HashMap<>(); - setting1Properties1.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); - setting1.add(setting3Properties1); + Map setting3Properties1 = new HashMap<>(); + setting3Properties1.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); + deploymentSettings.add(setting3Properties1); //Setting 4: Property 1 and 2 - HashMap setting4Properties1 = new HashMap<>(); + Map setting4Properties1 = new HashMap<>(); setting4Properties1.put(RepositorySetting.OVERRIDE_STRATEGY, RepositorySetting.OVERRIDE_STRATEGY_ALWAYS_APPLY); setting4Properties1.put(RepositorySetting.OPERATING_SYSTEM, "redhat7"); setting4Properties1.put(RepositorySetting.REPO_ID, "HDP"); setting4Properties1.put(RepositorySetting.BASE_URL, "http://localhost/repo"); - setting4.add(setting4Properties1); + repositorySettings.add(setting4Properties1); - HashMap setting4Properties2 = new HashMap<>(); + Map setting4Properties2 = new HashMap<>(); setting4Properties2.put(RepositorySetting.OVERRIDE_STRATEGY, RepositorySetting.OVERRIDE_STRATEGY_ALWAYS_APPLY); setting4Properties2.put(RepositorySetting.OPERATING_SYSTEM, "redhat7"); setting4Properties2.put(RepositorySetting.REPO_ID, "HDP-UTIL"); setting4Properties2.put(RepositorySetting.BASE_URL, "http://localhost/repo"); - setting4.add(setting4Properties2); + repositorySettings.add(setting4Properties2); + + properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, recoverySettings); + properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, serviceSettings); + properties.put(Setting.SETTING_NAME_COMPONENT_SETTINGS, componentSettings); + properties.put(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, deploymentSettings); + properties.put(Setting.SETTING_NAME_REPOSITORY_SETTINGS, repositorySettings); - properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, setting1); - properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, setting2); - properties.put(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, setting3); - properties.put(Setting.SETTING_NAME_REPOSITORY_SETTINGS, setting4); + setting = new Setting(properties); + } - Setting setting = new Setting(properties); + /** + * Test get and set of entire setting. + */ + @Test + public void testGetProperties() { assertEquals(properties, setting.getProperties()); } @@ -93,31 +150,47 @@ public void testGetProperties() { */ @Test public void testGetSettingProperties() { - Map>> properties = new HashMap<>(); - Set> setting1 = new HashSet<>(); - Set> setting2 = new HashSet<>(); + assertEquals(serviceSettings, setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS)); + } - // Setting 1: Property1 - HashMap setting1Properties1 = new HashMap<>(); - setting1Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "true"); - setting1.add(setting1Properties1); + @Test + public void recoveryEnabledAtComponentLevel() { + assertEquals("false", setting.getRecoveryEnabled("HDFS", "HDFS_CLIENT")); + assertEquals("true", setting.getRecoveryEnabled("HDFS", "NAMENODE")); + assertEquals("false", setting.getRecoveryEnabled("YARN", "YARN_CLIENT")); + assertEquals("true", setting.getRecoveryEnabled("YARN", "RESOURCE_MANAGER")); + } - // Setting 2: Property1 and Property2 - HashMap setting2Properties1 = new HashMap<>(); - setting2Properties1.put(Setting.SETTING_NAME_NAME, "HDFS"); - setting2Properties1.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); + @Test + public void recoveryEnabledAtServiceLevel() { + assertEquals("true", setting.getRecoveryEnabled("TEZ", "TEZ_CLIENT")); + assertEquals("false", setting.getRecoveryEnabled("HDFS", "DATANODE")); + } - HashMap setting2Properties2 = new HashMap<>(); - setting2Properties2.put(Setting.SETTING_NAME_NAME, "TEZ"); - setting2Properties2.put(Setting.SETTING_NAME_RECOVERY_ENABLED, "false"); + @Test + public void recoveryEnabledAtClusterLevel() { + assertEquals("true", setting.getRecoveryEnabled("OOZIE", "OOZIE_SERVER")); + assertEquals("true", setting.getRecoveryEnabled("YARN", "NODE_MANAGER")); + } - setting2.add(setting2Properties1); - setting2.add(setting2Properties2); + @Test + public void testAutoSkipFailureEnabled() { + Map skipFailureSetting = ImmutableMap.of(Setting.SETTING_NAME_SKIP_FAILURE, "true"); + Setting setting = new Setting(ImmutableMap.of(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, Collections.singleton(skipFailureSetting))); + assertTrue(setting.shouldSkipFailure()); + } - properties.put(Setting.SETTING_NAME_RECOVERY_SETTINGS, setting1); - properties.put(Setting.SETTING_NAME_SERVICE_SETTINGS, setting2); + @Test + public void testAutoSkipFailureDisabled() { + Map skipFailureSetting = ImmutableMap.of(Setting.SETTING_NAME_SKIP_FAILURE, "false"); + Setting setting = new Setting(ImmutableMap.of(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS, Collections.singleton(skipFailureSetting))); + assertFalse(setting.shouldSkipFailure()); + } - Setting setting = new Setting(properties); - assertEquals(setting2, setting.getSettingValue(Setting.SETTING_NAME_SERVICE_SETTINGS)); + @Test + public void testAutoSkipFailureUnspecified() { + Setting setting = new Setting(ImmutableMap.of()); + assertFalse(setting.shouldSkipFailure()); } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java new file mode 100644 index 00000000000..6235aae4666 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/StackComponentResolverTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; + +import java.util.stream.Stream; + +import org.apache.ambari.server.controller.internal.StackDefinition; +import org.apache.ambari.server.topology.validators.RejectUnknownComponents; +import org.apache.ambari.server.topology.validators.TopologyValidator; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; + +public class StackComponentResolverTest { + + private final TopologyValidator validator = new RejectUnknownComponents(); + private final ClusterTopology topology = createNiceMock(ClusterTopology.class); + private final StackDefinition stack = createNiceMock(StackDefinition.class); + + @Before + public void setUp() { + expect(topology.getStack()).andReturn(stack).anyTimes(); + } + + @After + public void tearDown() { + reset(topology, stack); + } + + @Test + public void acceptsKnownComponents() throws Exception { + // GIVEN + componentsInTopologyAre("VALID_COMPONENT", "ANOTHER_COMPONENT"); + validComponentsAre("VALID_COMPONENT", "ANOTHER_COMPONENT", "ONE_MORE_COMPONENT"); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + @Test(expected = InvalidTopologyException.class) + public void rejectsUnknownComponents() throws Exception { + // GIVEN + componentsInTopologyAre("VALID_COMPONENT", "UNKNOWN_COMPONENT"); + validComponentsAre("VALID_COMPONENT", "ANOTHER_COMPONENT"); + + // WHEN + validator.validate(topology); + } + + private void componentsInTopologyAre(String... components) { + expect(topology.getComponents()).andReturn(Stream.of(components) + .map(name -> ResolvedComponent.builder(new Component(name)).buildPartial()) + ).anyTimes(); + replay(topology); + } + + private void validComponentsAre(String... components) { + expect(stack.getComponents()).andReturn(ImmutableSet.builder().add(components).build()).anyTimes(); + replay(stack); + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index 7174a5e4f8e..9f3a1f04825 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -18,8 +18,10 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.capture; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; @@ -41,9 +43,11 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.stream.Stream; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.actionmanager.HostRoleStatus; @@ -69,6 +73,7 @@ import org.apache.ambari.server.state.quicklinksprofile.QuickLinksProfile; import org.apache.ambari.server.topology.tasks.ConfigureClusterTask; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.TopologyValidator; import org.apache.ambari.server.topology.validators.TopologyValidatorService; import org.easymock.Capture; import org.easymock.EasyMock; @@ -87,6 +92,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; /** @@ -109,6 +115,9 @@ public class TopologyManagerTest { @Rule public EasyMockRule mocks = new EasyMockRule(this); + @Mock + private org.apache.ambari.server.configuration.Configuration configuration; + @TestSubject private TopologyManager topologyManager = new TopologyManager(); @@ -170,6 +179,9 @@ public class TopologyManagerTest { @Mock private TopologyValidatorService topologyValidatorService; + @Mock + private ComponentResolver componentResolver; + private final Configuration stackConfig = new Configuration(new HashMap<>(), new HashMap<>()); private final Configuration bpConfiguration = new Configuration(new HashMap<>(), @@ -190,8 +202,10 @@ public class TopologyManagerTest { private HostGroupInfo group2Info = new HostGroupInfo("group2"); private Map groupInfoMap = new HashMap<>(); - private Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); - private Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group1Components = Arrays.asList(new Component("component1"), new Component("component2"), new Component("component3")); + private final Collection group1ComponentNames = group1Components.stream().map(Component::getName).collect(toSet()); + private final Collection group2Components = Arrays.asList(new Component("component3"), new Component("component4")); + private final Collection group2ComponentNames = group2Components.stream().map(Component::getName).collect(toSet()); private Map> group1ServiceComponents = new HashMap<>(); private Map> group2ServiceComponents = new HashMap<>(); @@ -209,6 +223,11 @@ public class TopologyManagerTest { @Before public void setup() throws Exception { + expect(configuration.getParallelTopologyTaskCreationThreadPoolSize()).andReturn(1).anyTimes(); + expect(configuration.getParallelStageExecution()).andReturn(false).anyTimes(); + expect(configuration.getGplLicenseAccepted()).andReturn(true).anyTimes(); + replay(configuration); + clusterTopologyCapture = newCapture(); configRequestPropertiesCapture = newCapture(); configRequestPropertiesCapture2 = newCapture(); @@ -247,21 +266,24 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(clusterTopologyMock.getComponents()).andReturn(Stream.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service2").buildPartial(), + ResolvedComponent.builder(new Component("component3")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service2").buildPartial() + )).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component3")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component4")).andReturn(Collections.singleton(group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service1")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - expect(blueprint.getHostGroupsForService("service2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); expect(blueprint.getName()).andReturn(BLUEPRINT_NAME).anyTimes(); - expect(blueprint.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopologyMock.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); + expect(clusterTopologyMock.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(STACK_ID)).anyTimes(); - expect(blueprint.getRepositorySettings()).andReturn(new ArrayList<>()).anyTimes(); + expect(blueprint.getSecurity()).andReturn(SecurityConfiguration.NONE).anyTimes(); + expect(blueprint.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); // don't expect toEntity() expect(stack.getAllConfigurationTypes("service1")).andReturn(Arrays.asList("service1-site", "service1-env")).anyTimes(); @@ -278,12 +300,17 @@ public void setup() throws Exception { expect(stack.getComponents("service1")).andReturn(components1).anyTimes(); expect(stack.getComponents("service2")).andReturn(components2).anyTimes(); expect(stack.getServiceForConfigType("service1-site")).andReturn("service1").anyTimes(); + expect(stack.getDependenciesForComponent(anyString())).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getServiceForConfigType("service2-site")).andReturn("service2").anyTimes(); expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes(); expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component2")).andReturn("service2").anyTimes(); + expect(stack.getServiceForComponent("component3")).andReturn("service1").anyTimes(); + expect(stack.getServiceForComponent("component4")).andReturn("service2").anyTimes(); expect(request.getBlueprint()).andReturn(blueprint).anyTimes(); expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes(); @@ -293,29 +320,24 @@ public void setup() throws Exception { expect(request.getHostGroupInfo()).andReturn(groupInfoMap).anyTimes(); expect(request.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); expect(request.getSecurityConfiguration()).andReturn(null).anyTimes(); + expect(request.getStackIds()).andReturn(ImmutableSet.of()).anyTimes(); + expect(request.getMpacks()).andReturn(ImmutableSet.of()).anyTimes(); + expect(componentResolver.resolveComponents(anyObject())).andReturn(ImmutableMap.of()).anyTimes(); - expect(group1.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent("group1")).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames()).andReturn(group1ComponentNames).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); - expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group1.getStack()).andReturn(stack).anyTimes(); - expect(group2.getBlueprintName()).andReturn(BLUEPRINT_NAME).anyTimes(); expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); - expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); + expect(clusterTopologyMock.containsMasterComponent("group2")).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); + expect(group2.getComponentNames()).andReturn(group2ComponentNames).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); - expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); - expect(group2.getStack()).andReturn(stack).anyTimes(); expect(logicalRequestFactory.createRequest(eq(1L), (TopologyRequest) anyObject(), capture(clusterTopologyCapture))). @@ -325,9 +347,10 @@ public void setup() throws Exception { expect(logicalRequest.getReservedHosts()).andReturn(Collections.singleton("host1")).anyTimes(); expect(logicalRequest.getRequestStatus()).andReturn(requestStatusResponse).anyTimes(); + expect(ambariContext.composeStacks(anyObject())).andReturn(stack).anyTimes(); expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes(); //todo: don't ignore param - ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), (SecurityType) isNull(), (String) isNull(), anyLong()); + ambariContext.createAmbariResources(isA(ClusterTopology.class), eq(CLUSTER_NAME), eq(SecurityType.NONE), isNull(), anyLong()); expectLastCall().anyTimes(); expect(ambariContext.getNextRequestId()).andReturn(1L).anyTimes(); expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes(); @@ -377,12 +400,12 @@ public void setup() throws Exception { @After public void tearDown() { PowerMock.verify(System.class); - verify(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + verify(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); PowerMock.reset(System.class); - reset(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + reset(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, logicalRequest, configurationRequest, configurationRequest2, configurationRequest3, requestStatusResponse, executor, persistedState, clusterTopologyMock, mockFuture, settingDAO); } @@ -538,7 +561,7 @@ private void requestFinished() { } private void replayAll() { - replay(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, + replay(blueprint, stack, request, group1, group2, ambariContext, logicalRequestFactory, componentResolver, configurationRequest, configurationRequest2, configurationRequest3, executor, persistedState, clusterTopologyMock, securityConfigurationFactory, credentialStoreService, clusterController, resourceProvider, mockFuture, requestStatusResponse, logicalRequest, settingDAO, @@ -547,7 +570,7 @@ private void replayAll() { @Test(expected = InvalidTopologyException.class) public void testScaleHosts__alreadyExistingHost() throws InvalidTopologyTemplateException, InvalidTopologyException, AmbariException, NoSuchStackException { - HashSet> propertySet = new HashSet<>(); + Set> propertySet = new HashSet<>(); Map properties = new TreeMap<>(); properties.put(HostResourceProvider.HOST_HOST_NAME_PROPERTY_ID, "host1"); properties.put(HostResourceProvider.HOST_GROUP_PROPERTY_ID, "group1"); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java index c2fea1d43e0..ae144266885 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java @@ -63,7 +63,7 @@ public void before() { EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(clusterConfigurationMock).anyTimes(); EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + EasyMock.expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); } @After @@ -94,7 +94,7 @@ public void testShouldValidationPassWhenAllConfigTypesAreValid() throws Exceptio clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -117,7 +117,7 @@ public void testShouldValidationFailWhenInvalidConfigGroupsSpecifiedInCCTemplate clusterRequestConfigTypes = new HashSet<>(Arrays.asList("oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); @@ -138,7 +138,7 @@ public void testShouldValidationFailWhenThereIsAnInvalidConfigGroupProvided() th clusterRequestConfigTypes = new HashSet<>(Arrays.asList("core-site", "yarn-site", "oozie-site")); EasyMock.expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(clusterRequestConfigTypes).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(new HashSet<>(Arrays.asList("YARN", "HDFS"))); EasyMock.expect(stackMock.getConfigurationTypes("HDFS")).andReturn(Arrays.asList("core-site")); EasyMock.expect(stackMock.getConfigurationTypes("YARN")).andReturn(Arrays.asList("yarn-site")); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java similarity index 86% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java index 46396877906..9e6ac130962 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/DependencyAndCardinalityValidatorTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; import static java.util.stream.Collectors.toList; import static org.easymock.EasyMock.expect; @@ -39,6 +39,13 @@ import org.apache.ambari.server.state.DependencyConditionInfo; import org.apache.ambari.server.state.DependencyInfo; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.Cardinality; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.InvalidTopologyException; import org.easymock.EasyMock; import org.easymock.EasyMockRule; import org.easymock.Mock; @@ -50,15 +57,15 @@ import com.google.common.collect.ImmutableSet; -/** - * BlueprintValidatorImpl unit tests. - */ -public class BlueprintValidatorImplTest { +public class DependencyAndCardinalityValidatorTest { private final Map hostGroups = new LinkedHashMap<>(); @Rule public EasyMockRule mocks = new EasyMockRule(this); + @Mock(type = MockType.NICE) + private ClusterTopology topology; + @Mock(type = MockType.NICE) private Blueprint blueprint; @@ -102,10 +109,12 @@ public void setup() { autoDeploy.setEnabled(true); autoDeploy.setCoLocate("service1/component2"); + expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); expect(blueprint.getStackIds()).andReturn(ImmutableSet.of(new StackId("HDP", "2.2"))).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); - expect(blueprint.getServices()).andReturn(services).anyTimes(); + expect(topology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(topology.getServices()).andReturn(services).anyTimes(); expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group1.getComponents()). @@ -127,7 +136,7 @@ public void setup() { dependenciesConditionInfos1.add(dependencyConditionInfo1); dependenciesConditionInfos1.add(dependencyConditionInfo2); - expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes(); + expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); } @After @@ -148,9 +157,10 @@ public void testValidateTopology_basic() throws Exception { expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); } @Test(expected = InvalidTopologyException.class) @@ -164,9 +174,10 @@ public void testValidateTopology_basic_negative() throws Exception { expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.emptyList()).anyTimes(); expect(blueprint.getHostGroupsForComponent("component2")).andReturn(Arrays.asList(group1, group2)).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); } @Test @@ -182,9 +193,9 @@ public void testValidateTopology_autoDeploy() throws Exception { expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); - replay(blueprint, stack, group1, group2, dependency1); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + replay(blueprint, topology, stack, group1, group2, dependency1); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); verify(group1); } @@ -219,10 +230,10 @@ public void testValidateTopology_autoDeploy_hasDependency() throws Exception { expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); expect(group1.addComponent(new Component("component3"))).andReturn(true).once(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); verify(group1); } @@ -239,7 +250,6 @@ public void testShouldDependencyBeExcludedWenRelatedServiceIsNotInBlueprint() th expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Cardinality cardinality = new Cardinality("1"); @@ -263,11 +273,11 @@ public void testShouldDependencyBeExcludedWenRelatedServiceIsNotInBlueprint() th expect(dependencyComponentInfo.isClient()).andReturn(true).anyTimes(); expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); @@ -285,7 +295,6 @@ public void testShouldThrowErrorWhenDependentComponentIsNotInBlueprint() throws expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Cardinality cardinality = new Cardinality("1"); @@ -306,11 +315,11 @@ public void testShouldThrowErrorWhenDependentComponentIsNotInBlueprint() throws expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); + replay(blueprint, topology, stack, group1, group2, dependency1, dependencyComponentInfo); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); @@ -329,7 +338,6 @@ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIs expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes(); - expect(blueprint.getName()).andReturn("blueprint-1").anyTimes(); Map> properties = new HashMap<>(); Map typeProps = new HashMap<>(); typeProps.put("yarn.resourcemanager.hostname", "testhost"); @@ -368,11 +376,11 @@ public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIs expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes(); expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes(); - replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2); + replay(blueprint, topology, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2); // WHEN - BlueprintValidator validator = new BlueprintValidatorImpl(null); - validator.validateTopology(blueprint); + TopologyValidator validator = new DependencyAndCardinalityValidator(); + validator.validate(topology); // THEN verify(group1); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java new file mode 100644 index 00000000000..2cb1ef10ed4 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/GplPropertiesValidatorTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.CODEC_CLASSES_PROPERTY_NAME; +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.LZO_CODEC_CLASS; +import static org.apache.ambari.server.topology.validators.GplPropertiesValidator.LZO_CODEC_CLASS_PROPERTY_NAME; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.reset; +import static org.easymock.EasyMock.verify; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +public class GplPropertiesValidatorTest { + + private static final boolean ACCEPTED = true; + + private final org.apache.ambari.server.configuration.Configuration serverConfig = createNiceMock(org.apache.ambari.server.configuration.Configuration.class); + private final TopologyValidator validator = new GplPropertiesValidator(serverConfig); + + @Before + public void setup() { + reset(serverConfig); + } + + @After + public void tearDown() { + verify(serverConfig); + } + + @Test(expected = GPLLicenseNotAcceptedException.class) // THEN + public void rejectsCodecsPropertyWithLzoClassIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + CODEC_CLASSES_PROPERTY_NAME, "OtherCodec, " + LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + } + + @Test(expected = GPLLicenseNotAcceptedException.class) // THEN + public void rejectsLzoCodecPropertyIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + LZO_CODEC_CLASS_PROPERTY_NAME, LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + } + + @Test + public void allowsLzoCodecIfGplIsAccepted() throws Exception { + // GIVEN + gpl(ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + LZO_CODEC_CLASS_PROPERTY_NAME, LZO_CODEC_CLASS, + CODEC_CLASSES_PROPERTY_NAME, "OtherCodec," + LZO_CODEC_CLASS + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + @Test + public void allowsConfigWithoutReferenceToGplEvenIfGplIsNotAccepted() throws Exception { + // GIVEN + gpl(!ACCEPTED); + ClusterTopology topology = TopologyValidatorTests.topologyWithProperties(ImmutableMap.of( + "core-site", ImmutableMap.of( + "fs.defaultFS", "hdfs://localhost:8020", + "io.compression.codecs", "org.apache.hadoop.io.compress.DefaultCodec" + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exception expected + } + + private void gpl(boolean accepted) { + expect(serverConfig.getGplLicenseAccepted()).andReturn(accepted).atLeastOnce(); + replay(serverConfig); + } +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index 3ce0a024813..805edaddf5b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -14,15 +14,16 @@ package org.apache.ambari.server.topology.validators; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import org.apache.ambari.server.topology.Blueprint; import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.InvalidTopologyException; -import org.easymock.EasyMock; +import org.apache.ambari.server.topology.ResolvedComponent; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -32,27 +33,25 @@ import org.junit.Rule; import org.junit.Test; +import com.google.common.collect.ImmutableSet; + public class HiveServiceValidatorTest extends EasyMockSupport { @Rule public EasyMockRule mocks = new EasyMockRule(this); @Mock - private ClusterTopology clusterTopologyMock; - - @Mock - private Blueprint blueprintMock; + private ClusterTopology topology; @Mock - private Configuration configurationMock; + private Configuration configuration; @TestSubject - private HiveServiceValidator hiveServiceValidator = new HiveServiceValidator(); + private final TopologyValidator hiveServiceValidator = new HiveServiceValidator(); @Before public void setUp() throws Exception { - - + expect(topology.getConfiguration()).andReturn(configuration).anyTimes(); } @After @@ -61,101 +60,90 @@ public void tearDown() throws Exception { } @Test - public void testShouldValidationPassWhenHiveServiceIsNotInBlueprint() throws Exception { - + public void allowsTopologyWithoutHive() throws Exception { // GIVEN - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(Collections.emptySet()); - replayAll(); + noHiveInTopology(); + noHiveConfig(); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenHiveServiceIsMissingConfigType() throws Exception { - + @Test + public void allowsExistingDatabaseWithoutMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "OOZIE"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(Collections.emptySet()); - - replayAll(); + topologyHasMysql(false); + hiveDatabaseIs("Existing MySQL"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } @Test - public void testShouldValidationPassWhenCustomHiveDatabaseSettingsProvided() throws Exception { - + public void allowsNewMysqlDatabaseWithMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "OOZIE"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("PSQL"); - replayAll(); + topologyHasMysql(true); + hiveDatabaseIs("New MySQL Database"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); // THEN - + // OK } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenDefaultsAreUsedAndMysqlComponentIsMissing() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsHiveWithoutConfig() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "HDFS"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(Collections.emptyList()).anyTimes(); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database"); - replayAll(); - + topologyHasMysql(true); + noHiveConfig(); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); - - // THEN - + hiveServiceValidator.validate(topology); } - @Test - public void testShouldValidationPassWhenDefaultsAreUsedAndMsqlComponentIsListed() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsNewMysqlDatabaseWithoutMysqlComponent() throws Exception { // GIVEN - Collection blueprintServices = Arrays.asList("HIVE", "HDFS", "MYSQL_SERVER"); - Collection hiveComponents = Arrays.asList("MYSQL_SERVER"); - Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); - EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(hiveComponents).anyTimes(); - EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); - EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); - - EasyMock.expect(configurationMock.getPropertyValue("hive-env", "hive_database")).andReturn("New MySQL Database"); - replayAll(); + topologyHasMysql(false); + hiveDatabaseIs("New MySQL Database"); // WHEN - hiveServiceValidator.validate(clusterTopologyMock); + hiveServiceValidator.validate(topology); + } - // THEN + private void noHiveConfig() { + expect(configuration.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "hadoop-env")).anyTimes(); + replay(configuration); + } + + private void hiveDatabaseIs(String database) { + expect(configuration.getAllConfigTypes()).andReturn(Arrays.asList("hive-env", "hive-site", "core-site", "hadoop-env")); + expect(configuration.getPropertyValue("hive-env", "hive_database")).andReturn(database).anyTimes(); + replay(configuration); + } + private void topologyHasMysql(boolean hasMysql) { + ImmutableSet.Builder components = ImmutableSet.builder().add("HIVE_CLIENT", "HIVE_METASTORE", "HIVE_SERVER"); + if (hasMysql) { + components.add("MYSQL_SERVER"); + } + expect(topology.getComponents()).andReturn(components.build().stream() + .map(name -> ResolvedComponent.builder(new Component(name)).serviceType("HIVE").buildPartial()) + ).anyTimes(); + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN", "HIVE")).anyTimes(); + replay(topology); } + + private void noHiveInTopology() { + expect(topology.getServices()).andReturn(ImmutableSet.of("HDFS", "YARN")).anyTimes(); + replay(topology); + } + } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java new file mode 100644 index 00000000000..48cd6718308 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RejectUnknownStacksTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.easymock.EasyMockSupport; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; +import com.google.inject.util.Providers; + +public class RejectUnknownStacksTest extends EasyMockSupport { + + private final StackId validStackId = new StackId("valid", "1.0"); + private final StackId anotherValidStackId = new StackId("another", "2.1"); + private final StackId invalidStackId = new StackId("invalid", "3.2"); + private final StackId anotherInvalidStackId = new StackId("invalid", "1.1"); + private RejectUnknownStacks validator; + + @Before + public void setUp() { + AmbariMetaInfo metaInfo = createNiceMock(AmbariMetaInfo.class); + validator = new RejectUnknownStacks(Providers.of(metaInfo)); + + expect(metaInfo.isKnownStack(validStackId)).andReturn(true).anyTimes(); + expect(metaInfo.isKnownStack(anotherValidStackId)).andReturn(true).anyTimes(); + expect(metaInfo.isKnownStack(invalidStackId)).andReturn(false).anyTimes(); + expect(metaInfo.isKnownStack(anotherInvalidStackId)).andReturn(false).anyTimes(); + } + + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectsUnknownStack() throws InvalidTopologyException { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(validStackId, invalidStackId)).anyTimes(); + replayAll(); + + // WHEN + validator.validate(topology); + } + + @Test + public void acceptsKnownStack() throws InvalidTopologyException { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(validStackId, anotherValidStackId)).anyTimes(); + replayAll(); + + // WHEN + validator.validate(topology); + + // THEN + // no exception + } + + @Test + public void reportsUnknownStacks() { + // GIVEN + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getStackIds()).andReturn(ImmutableSet.of(invalidStackId, anotherInvalidStackId)).anyTimes(); + replayAll(); + + // WHEN + try { + validator.validate(topology); + fail("Expected " + InvalidTopologyException.class); + } catch (InvalidTopologyException e) { + // THEN + assertTrue(e.getMessage(), e.getMessage().contains(invalidStackId.toString())); + assertTrue(e.getMessage(), e.getMessage().contains(anotherInvalidStackId.toString())); + } + } +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java index 780ca53bdd9..e249feb8da6 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredConfigPropertiesValidatorTest.java @@ -99,17 +99,15 @@ public void setup() { EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); EasyMock.expect(blueprintMock.getHostGroups()).andReturn(hostGroups); - EasyMock.expect(blueprintMock.getServices()).andReturn(bpServices); - EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + EasyMock.expect(clusterTopologyMock.getServices()).andReturn(bpServices); + EasyMock.expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); EasyMock.expect(masterHostGroupMock.getName()).andReturn("master").anyTimes(); EasyMock.expect(masterHostGroupMock.getConfiguration()).andReturn(masterHostGroupConfigurationMock).anyTimes(); - EasyMock.expect(masterHostGroupMock.getServices()).andReturn(masterHostGroupServices); EasyMock.expect(slaveHostGroupMock.getName()).andReturn("slave").anyTimes(); EasyMock.expect(slaveHostGroupMock.getConfiguration()).andReturn(slaveHostGroupConfigurationMock).anyTimes(); - EasyMock.expect(slaveHostGroupMock.getServices()).andReturn(slaveHostGroupServices); // there are 2 hostgroups to be considered by the test hostGroups.put("master", masterHostGroupMock); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java similarity index 87% rename from ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java rename to ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java index 722f5f1e31c..f7ce215ccd8 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/RequiredPasswordValidatorTest.java @@ -16,21 +16,28 @@ * limitations under the License. */ -package org.apache.ambari.server.topology; +package org.apache.ambari.server.topology.validators; import static junit.framework.Assert.assertEquals; import static org.easymock.EasyMock.expect; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.stream.Stream; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.state.PropertyInfo; -import org.apache.ambari.server.topology.validators.RequiredPasswordValidator; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Component; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.HostGroup; +import org.apache.ambari.server.topology.HostGroupInfo; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.ResolvedComponent; import org.easymock.EasyMockRule; import org.easymock.EasyMockSupport; import org.easymock.Mock; @@ -74,8 +81,6 @@ public class RequiredPasswordValidatorTest extends EasyMockSupport { private static final Map hostGroups = new HashMap<>(); private static final Map hostGroupInfo = new HashMap<>(); - private static final Collection group1Components = new HashSet<>(); - private static final Collection group2Components = new HashSet<>(); private static final Collection service1Components = new HashSet<>(); private static final Collection service2Components = new HashSet<>(); private static final Collection service3Components = new HashSet<>(); @@ -119,13 +124,6 @@ public void setup() { hostGroups.put("group1", group1); hostGroups.put("group2", group2); - group1Components.add("component1"); - group1Components.add("component2"); - group1Components.add("component3"); - - group2Components.add("component1"); - group2Components.add("component4"); - service1Components.add("component1"); service1Components.add("component2"); service2Components.add("component3"); @@ -145,21 +143,17 @@ public void setup() { expect(blueprint.getHostGroups()).andReturn(hostGroups).anyTimes(); expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); - - expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); - expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); - expect(group1.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes(); - expect(group1.getComponentNames("service2")).andReturn(Arrays.asList("component3")).anyTimes(); - expect(group1.getComponentNames("service3")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponentNames("service1")).andReturn(Arrays.asList("component1")).anyTimes(); - expect(group2.getComponentNames("service2")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponentNames("service3")).andReturn(Arrays.asList("component4")).anyTimes(); - - expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); - expect(stack.getServiceForComponent("component2")).andReturn("service1").anyTimes(); - expect(stack.getServiceForComponent("component3")).andReturn("service2").anyTimes(); - expect(stack.getServiceForComponent("component4")).andReturn("service3").anyTimes(); + expect(topology.getStack()).andReturn(stack).anyTimes(); + + expect(topology.getComponentsInHostGroup("group1")).andReturn(Stream.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component2")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component3")).serviceType("service2").buildPartial() + )).anyTimes(); + expect(topology.getComponentsInHostGroup("group2")).andReturn(Stream.of( + ResolvedComponent.builder(new Component("component1")).serviceType("service1").buildPartial(), + ResolvedComponent.builder(new Component("component4")).serviceType("service3").buildPartial() + )).anyTimes(); expect(stack.getRequiredConfigurationProperties("service1", PropertyInfo.PropertyType.PASSWORD)).andReturn(service1RequiredPwdConfigs).anyTimes(); expect(stack.getRequiredConfigurationProperties("service2", PropertyInfo.PropertyType.PASSWORD)).andReturn(service2RequiredPwdConfigs).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java new file mode 100644 index 00000000000..fa3021f047e --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/SecretReferenceValidatorTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.apache.ambari.server.topology.validators.TopologyValidatorTests.topologyWithProperties; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.junit.Test; + +import com.google.common.collect.ImmutableMap; + +public class SecretReferenceValidatorTest { + + private final TopologyValidator validator = new SecretReferenceValidator(); + + @Test + public void acceptsTopologyWithoutSecretReferences() throws InvalidTopologyException { + ClusterTopology topology = topologyWithProperties(ImmutableMap.of( + "hdfs-site", ImmutableMap.of( + "password", "secret" + ) + )); + + // WHEN + validator.validate(topology); + + // THEN + // no exceptions expected + } + + @Test(expected = InvalidTopologyException.class) + public void rejectsTopologyWithSecretReferences() throws InvalidTopologyException { + ClusterTopology topology = topologyWithProperties(ImmutableMap.of( + "hdfs-site", ImmutableMap.of( + "password", "SECRET:hdfs-site:1:test" + ) + )); + + // WHEN + validator.validate(topology); + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java index 417c403e1fd..a03290a2ff0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/StackConfigTypeValidatorTest.java @@ -18,8 +18,7 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import org.apache.ambari.server.controller.internal.Stack; import org.apache.ambari.server.topology.Blueprint; @@ -55,8 +54,6 @@ public class StackConfigTypeValidatorTest extends EasyMockSupport { @Mock private ClusterTopology clusterTopologyMock; - private Set clusterRequestConfigTypes; - @TestSubject private StackConfigTypeValidator stackConfigTypeValidator = new StackConfigTypeValidator(); @@ -65,7 +62,7 @@ public void before() { expect(clusterTopologyMock.getConfiguration()).andReturn(clusterConfigurationMock).anyTimes(); expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); - expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes(); + expect(clusterTopologyMock.getStack()).andReturn(stackMock).anyTimes(); } @After @@ -74,30 +71,24 @@ public void after() { } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailWhenUnknownConfigTypeComesIn() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectInvalidType() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("invalid-site"))); - + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Collections.singleton("invalid-site")); replayAll(); // WHEN stackConfigTypeValidator.validate(clusterTopologyMock); - - // THEN - // exception is thrown - } @Test - public void testShouldValidationPassifNoConfigTypesomeIn() throws Exception { + public void allowEmptyConfig() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Collections.emptyList())); - + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Collections.emptySet()); replayAll(); // WHEN @@ -105,16 +96,27 @@ public void testShouldValidationPassifNoConfigTypesomeIn() throws Exception { // THEN // no exception is thrown - } - @Test(expected = InvalidTopologyException.class) - public void testShouldValidationFailIfMultipleInvalidConfigTypesComeIn() throws Exception { + @Test(expected = InvalidTopologyException.class) // THEN + public void rejectMultipleInvalidTypes() throws Exception { // GIVEN expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); - expect(stackConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("core-site", "yarn-site"))); - expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(new HashSet<>(Arrays.asList("invalid-site-1", "invalid-default"))); + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("core-site", "yarn-site")); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(Arrays.asList("invalid-site-1", "invalid-default")); + replayAll(); + // WHEN + stackConfigTypeValidator.validate(clusterTopologyMock); + } + + @Test + public void allowValidTypes() throws InvalidTopologyException { + // GIVEN + List configTypes = Arrays.asList("core-site", "yarn-site"); + expect(stackMock.getConfiguration()).andReturn(stackConfigurationMock); + expect(stackConfigurationMock.getAllConfigTypes()).andReturn(configTypes); + expect(clusterConfigurationMock.getAllConfigTypes()).andReturn(configTypes); replayAll(); // WHEN @@ -122,6 +124,5 @@ public void testShouldValidationFailIfMultipleInvalidConfigTypesComeIn() throws // THEN // no exception is thrown - } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java new file mode 100644 index 00000000000..2ae84412de1 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/TopologyValidatorTests.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Configuration; + +/** + * Helper for TopologyValidator tests. + */ +public class TopologyValidatorTests { + + /** + * Creates a mock ClusterTopology with the given properties. + */ + static ClusterTopology topologyWithProperties(Map> properties) { + Configuration topologyConfig = new Configuration(properties, new HashMap<>()); + ClusterTopology topology = createNiceMock(ClusterTopology.class); + expect(topology.getConfiguration()).andReturn(topologyConfig).anyTimes(); + replay(topology); + return topology; + } + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java index 015dc36ceb8..926bbd335e2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java @@ -76,7 +76,7 @@ public void skipsValidatingIrrelevantProperty() throws Exception { public void setUp() throws Exception { expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); expect(clusterTopology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); - expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(clusterTopology.getStack()).andReturn(stack).anyTimes(); expect(stack.getConfigurationPropertiesWithMetadata(SERVICE, CONFIG_TYPE)).andReturn(stackConfigWithMetadata).anyTimes(); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index d43a712bf92..18637d4505e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -82,9 +82,15 @@ import org.apache.ambari.server.state.cluster.ClusterFactory; import org.apache.ambari.server.state.host.HostFactory; import org.apache.ambari.server.state.stack.OsFamily; +import org.apache.ambari.server.topology.ComponentResolver; +import org.apache.ambari.server.topology.DefaultStackFactory; import org.apache.ambari.server.topology.PersistedState; +import org.apache.ambari.server.topology.StackComponentResolver; +import org.apache.ambari.server.topology.StackFactory; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.topology.tasks.ConfigureClusterTaskFactory; +import org.apache.ambari.server.topology.validators.BasicBlueprintValidator; +import org.apache.ambari.server.topology.validators.BlueprintValidator; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; import org.easymock.EasyMockSupport; @@ -135,6 +141,9 @@ protected void configure() { bind(MpackManagerFactory.class).toInstance(createNiceMock(MpackManagerFactory.class)); bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class)); bind(RootLevelSettingsManagerFactory.class).toInstance(createNiceMock(RootLevelSettingsManagerFactory.class)); + bind(ComponentResolver.class).to(StackComponentResolver.class); + bind(BlueprintValidator.class).to(BasicBlueprintValidator.class); + bind(StackFactory.class).to(DefaultStackFactory.class); install(new FactoryModuleBuilder().build(ExecutionCommandWrapperFactory.class)); install(new FactoryModuleBuilder().implement(Config.class, ConfigImpl.class).build(ConfigFactory.class)); install(new FactoryModuleBuilder().build(ConfigureClusterTaskFactory.class));