diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java index acdf9edb7d9..6be0ff221e5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java @@ -59,7 +59,6 @@ import org.apache.ambari.server.topology.AmbariContext; import org.apache.ambari.server.topology.ClusterTopology; import org.apache.ambari.server.topology.ClusterTopologyImpl; -import org.apache.ambari.server.topology.Component; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; @@ -445,10 +444,10 @@ private List> formatGroupsAsList(ClusterTopology topology) { */ private List> processHostGroupComponents(HostGroup group) { List> listHostGroupComponents = new ArrayList<>(); - for (Component component : group.getComponents()) { + for (String component : group.getComponentNames()) { Map mapComponentProperties = new HashMap<>(); listHostGroupComponents.add(mapComponentProperties); - mapComponentProperties.put("name", component.getName()); + mapComponentProperties.put("name", component); } return listHostGroupComponents; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java index 273c0ff1949..ddd746b2cd8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessor.java @@ -89,7 +89,7 @@ public void adviseConfiguration(ClusterTopology clusterTopology, Map> hgComponentsMap = gatherHostGroupComponents(clusterTopology); Map> hgHostsMap = gatherHostGroupBindings(clusterTopology); Map> componentHostsMap = gatherComponentsHostsMap(hgComponentsMap, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java index 9de89ee9a67..5bad0f246eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java @@ -941,8 +941,7 @@ public void performStaticInjection() { StackDefinedPropertyProvider.init(injector); AbstractControllerResourceProvider.init(injector.getInstance(ResourceProviderFactory.class)); BlueprintResourceProvider.init(injector.getInstance(BlueprintFactory.class), - injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class), - injector.getInstance(Gson.class), ambariMetaInfo); + injector.getInstance(BlueprintDAO.class), injector.getInstance(SecurityConfigurationFactory.class), ambariMetaInfo); StackDependencyResourceProvider.init(ambariMetaInfo); ClusterResourceProvider.init(injector.getInstance(TopologyManager.class), injector.getInstance(TopologyRequestFactoryImpl.class), injector.getInstance(SecurityConfigurationFactory diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java index 7eef2dfed42..1a4221e378e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintResourceProvider.java @@ -18,7 +18,12 @@ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.toList; + +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -57,17 +62,18 @@ import org.apache.ambari.server.topology.BlueprintFactory; import org.apache.ambari.server.topology.GPLLicenseNotAcceptedException; import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.SecurityConfiguration; import org.apache.ambari.server.topology.SecurityConfigurationFactory; import org.apache.ambari.server.utils.SecretReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import com.google.gson.Gson; /** @@ -101,6 +107,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide public static final String COMPONENT_PROPERTY_ID ="components"; public static final String COMPONENT_NAME_PROPERTY_ID ="name"; public static final String COMPONENT_PROVISION_ACTION_PROPERTY_ID = "provision_action"; + protected static final String COMPONENT_MPACK_INSTANCE_PROPERTY = "mpack_instance"; + protected static final String COMPONENT_SERVICE_INSTANCE_PROPERTY = "service_instance"; // Configurations public static final String CONFIGURATION_PROPERTY_ID = "configurations"; @@ -120,6 +128,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide "Configuration elements must be Maps"; public static final String CONFIGURATION_MAP_SIZE_CHECK_ERROR_MESSAGE = "Configuration Maps must hold a single configuration type each"; + public static final String MPACK_INSTANCES_PROPERTY_ID = "mpack_instances"; + + // Primary Key Fields + private static Set pkPropertyIds = + new HashSet<>(Arrays.asList(new String[]{ + BLUEPRINT_NAME_PROPERTY_ID})); /** * The key property ids for a Blueprint resource. @@ -138,7 +152,8 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide BLUEPRINT_SECURITY_PROPERTY_ID, HOST_GROUP_PROPERTY_ID, CONFIGURATION_PROPERTY_ID, - SETTING_PROPERTY_ID); + SETTING_PROPERTY_ID, + MPACK_INSTANCES_PROPERTY_ID); /** * Used to create Blueprint instances @@ -158,7 +173,7 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide /** * Used to serialize to/from json. */ - private static Gson jsonSerializer; + private static ObjectMapper jsonSerializer = new ObjectMapper(); // ----- Constructors ---------------------------------------------------- @@ -176,14 +191,12 @@ public class BlueprintResourceProvider extends AbstractControllerResourceProvide * * @param factory blueprint factory * @param dao blueprint data access object - * @param gson json serializer */ public static void init(BlueprintFactory factory, BlueprintDAO dao, SecurityConfigurationFactory - securityFactory, Gson gson, AmbariMetaInfo metaInfo) { + securityFactory, AmbariMetaInfo metaInfo) { blueprintFactory = factory; blueprintDAO = dao; securityConfigurationFactory = securityFactory; - jsonSerializer = gson; ambariMetaInfo = metaInfo; } @@ -325,10 +338,16 @@ protected Resource toResource(BlueprintEntity entity, Set requestedIds) for (HostGroupComponentEntity component : components) { Map mapComponentProps = new HashMap<>(); mapComponentProps.put(COMPONENT_NAME_PROPERTY_ID, component.getName()); - if (component.getProvisionAction() != null) { mapComponentProps.put(COMPONENT_PROVISION_ACTION_PROPERTY_ID, component.getProvisionAction().toString()); } + if (component.getMpackName() != null) { + mapComponentProps.put(COMPONENT_MPACK_INSTANCE_PROPERTY, + component.getMpackName() + "-" + component.getMpackVersion()); + } + if (component.getServiceName() != null) { + mapComponentProps.put(COMPONENT_SERVICE_INSTANCE_PROPERTY, component.getServiceName()); + } listComponentProps.add(mapComponentProps); } @@ -351,9 +370,37 @@ protected Resource toResource(BlueprintEntity entity, Set requestedIds) setResourceProperty(resource, BLUEPRINT_SECURITY_PROPERTY_ID, securityConfigMap, requestedIds); } + Collection> mpacks = entity.getMpackInstances().stream().map(mpackEntity -> { + MpackInstance mpack = MpackInstance.fromEntity(mpackEntity); + Map mpackAsMap = fromJson(toJson(mpack), Map.class); + return mpackAsMap; + } ).collect(toList()); + setResourceProperty(resource, MPACK_INSTANCES_PROPERTY_ID, mpacks, requestedIds); + return resource; } + private static T fromJson(String json, Class valueType) { + if (null == json) { + return null; + } + try { + return jsonSerializer.readValue(json, valueType); + } + catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + private static String toJson(Object object) { + try { + return jsonSerializer.writeValueAsString(object); + } + catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + /** * Populate a list of configuration property maps from a collection of configuration entities. * @@ -371,9 +418,7 @@ List>> populateConfigurationList( String type = config.getType(); if(config instanceof BlueprintConfigEntity) { - Map properties = jsonSerializer.>fromJson( - config.getConfigData(), Map.class); - + Map properties = fromJson(config.getConfigData(), Map.class); // TODO: use multiple mpacks BlueprintMpackInstanceEntity mpack = @@ -387,19 +432,17 @@ List>> populateConfigurationList( } Map> propertiesTypes = - metaInfoStack.getConfigPropertiesTypes(type); + metaInfoStack.getConfigPropertiesTypes(type); SecretReference.replacePasswordsWithReferences(propertiesTypes, properties, type, -1l); configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); } else { - Map properties = jsonSerializer.>fromJson( - config.getConfigData(), Map.class); + Map properties = fromJson(config.getConfigData(), Map.class); configTypeDefinition.put(PROPERTIES_PROPERTY_ID, properties); } - Map> attributes = jsonSerializer.>>fromJson( - config.getConfigAttributes(), Map.class); + Map> attributes = fromJson(config.getConfigAttributes(), Map.class); if (attributes != null && !attributes.isEmpty()) { configTypeDefinition.put(PROPERTIES_ATTRIBUTES_PROPERTY_ID, attributes); } @@ -423,8 +466,7 @@ public static List> populateSettingList( if (settings != null) { for (BlueprintSettingEntity setting : settings) { - List> propertiesList = jsonSerializer.>>fromJson( - setting.getSettingData(), List.class); + List> propertiesList = fromJson(setting.getSettingData(), List.class); Map settingMap = new HashMap<>(); settingMap.put(setting.getSettingName(), propertiesList); listSettings.add(settingMap); @@ -503,7 +545,7 @@ public Void invoke() throws AmbariException { String rawRequestBody = requestInfoProps.get(Request.REQUEST_INFO_BODY_PROPERTY); Preconditions.checkArgument(!Strings.isNullOrEmpty(rawRequestBody), REQUEST_BODY_EMPTY_ERROR_MESSAGE); - Map rawBodyMap = jsonSerializer.>fromJson(rawRequestBody, Map.class); + Map rawBodyMap = fromJson(rawRequestBody, Map.class); Object configurationData = rawBodyMap.get(CONFIGURATION_PROPERTY_ID); if (configurationData != null) { @@ -545,8 +587,7 @@ public Void invoke() throws AmbariException { } LOG.info("Creating Blueprint, name=" + blueprint.getName()); - String blueprintSetting = blueprint.getSetting() == null ? "(null)" : - jsonSerializer.toJson(blueprint.getSetting().getProperties()); + String blueprintSetting = blueprint.getSetting() == null ? "(null)" : toJson(blueprint.getSetting().getProperties()); LOG.info("Blueprint setting=" + blueprintSetting); try { @@ -588,8 +629,8 @@ public void applyConfiguration(Map configuration, BlueprintConfi } } - blueprintConfiguration.setConfigData(jsonSerializer.toJson(configData)); - blueprintConfiguration.setConfigAttributes(jsonSerializer.toJson(configAttributes)); + blueprintConfiguration.setConfigData(toJson(configData)); + blueprintConfiguration.setConfigAttributes(toJson(configAttributes)); } protected abstract void addProperty(Map configData, diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java index 22960e16a33..7a1f517d4e8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterResourceProvider.java @@ -86,6 +86,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider public static final String HEALTH_REPORT = "health_report"; public static final String CREDENTIAL_STORE_PROPERTIES = "credential_store_properties"; public static final String REPO_VERSION = "repository_version"; + public static final String MPACK_INSTANCES = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID; public static final String CLUSTER_ID_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_ID; public static final String CLUSTER_NAME_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + CLUSTER_NAME; public static final String CLUSTER_VERSION_PROPERTY_ID = RESPONSE_KEY + PropertyHelper.EXTERNAL_PATH_SEP + VERSION; @@ -163,6 +164,7 @@ public class ClusterResourceProvider extends AbstractControllerResourceProvider propertyIds.add(SECURITY); propertyIds.add(CREDENTIALS); propertyIds.add(QUICKLINKS_PROFILE); + propertyIds.add(MPACK_INSTANCES); propertyIds.add(CLUSTER_STATE_PROPERTY_ID); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java index 16d3114949f..ed6b5fd04b9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequest.java @@ -132,10 +132,10 @@ private void createBlueprint(Collection exportedHostGroups, S componentList.add(new Component(component)); } - hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, stack, componentList, + hostGroups.add(new HostGroupImpl(exportedHostGroup.getName(), bpName, Collections.singleton(stack), componentList, exportedHostGroup.getConfiguration(), String.valueOf(exportedHostGroup.getCardinality()))); } - blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null); + blueprint = new BlueprintImpl(bpName, hostGroups, stack, configuration, null, null); } private void createHostGroupInfo(Collection exportedHostGroups) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java index 1fd60917bea..115973c9ec5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ProvisionClusterRequest.java @@ -17,6 +17,9 @@ */ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.toList; + +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -34,11 +37,16 @@ import org.apache.ambari.server.topology.Credential; import org.apache.ambari.server.topology.HostGroupInfo; import org.apache.ambari.server.topology.InvalidTopologyTemplateException; +import org.apache.ambari.server.topology.ManagementPackMapping; +import org.apache.ambari.server.topology.MpackInstance; import org.apache.ambari.server.topology.NoSuchBlueprintException; import org.apache.ambari.server.topology.SecurityConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Enums; import com.google.common.base.Optional; import com.google.common.base.Strings; @@ -119,6 +127,13 @@ public class ProvisionClusterRequest extends BaseClusterRequest { */ public static final String QUICKLINKS_PROFILE_SERVICES_PROPERTY = "quicklinks_profile/services"; + public static final String MANAGEMENT_PACK_MAPPINGS_PROPERTY = "management_pack_mappings"; + + public static final String MPACK_INSTANCES_PROPERTY = BlueprintResourceProvider.MPACK_INSTANCES_PROPERTY_ID; + + public static final String MPACK_INSTANCE_PROPERTY = "mpack_instance"; + + public static final String COMPONENT_NAME_PROPERTY = "component_name"; /** * configuration factory @@ -148,6 +163,8 @@ public class ProvisionClusterRequest extends BaseClusterRequest { private final String quickLinksProfileJson; + private Collection mpackInstances; + private final static Logger LOG = LoggerFactory.getLogger(ProvisionClusterRequest.class); /** @@ -196,6 +213,8 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura setProvisionAction(parseProvisionAction(properties)); + processMpackInstances(properties); + try { this.quickLinksProfileJson = processQuickLinksProfile(properties); } catch (QuickLinksProfileEvaluationException ex) { @@ -203,6 +222,21 @@ public ProvisionClusterRequest(Map properties, SecurityConfigura } } + private void processMpackInstances(Map properties) throws InvalidTopologyTemplateException { + if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) { + ObjectMapper mapper = new ObjectMapper(); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + try { + String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY)); + this.mpackInstances = mapper.readValue(mpackInstancesJson, + new TypeReference>() {}); + } + catch (IOException ex) { + throw new InvalidTopologyTemplateException("Cannot process mpack instances.", ex); + } + } + } + private String processQuickLinksProfile(Map properties) throws QuickLinksProfileEvaluationException { Object globalFilters = properties.get(QUICKLINKS_PROFILE_FILTERS_PROPERTY); Object serviceFilters = properties.get(QUICKLINKS_PROFILE_SERVICES_PROPERTY); @@ -339,6 +373,16 @@ private void processHostGroup(Map hostGroupProperties) throws In processGroupHosts(name, (Collection>) hostGroupProperties.get(HOSTGROUP_HOSTS_PROPERTY), hostGroupInfo); + // process mpack mappings + if (hostGroupProperties.containsKey(MANAGEMENT_PACK_MAPPINGS_PROPERTY)) { + Set> mpackMappingsRaw = + (Set>)hostGroupProperties.get(MANAGEMENT_PACK_MAPPINGS_PROPERTY); + Collection mpackMappings = mpackMappingsRaw.stream(). + map(mapping -> new ManagementPackMapping(mapping.get(COMPONENT_NAME_PROPERTY), mapping.get(MPACK_INSTANCE_PROPERTY))). + collect(toList()); + hostGroupInfo.setManagementPackMappings(mpackMappings); + } + // don't set the parent configuration hostGroupInfo.setConfiguration(configurationFactory.getConfiguration( (Collection>) hostGroupProperties.get(CONFIGURATIONS_PROPERTY))); @@ -490,4 +534,7 @@ public String getDefaultPassword() { return defaultPassword; } + public Collection getMpackInstances() { + return mpackInstances; + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index a8fff2efbd4..b54dd20467a 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -659,6 +659,7 @@ private void parseComponents(String service) throws AmbariException{ if (componentDependencies != null && ! componentDependencies.isEmpty()) { dependencies.put(componentName, componentDependencies); } + if (component.isMaster()) { masterComponents.add(componentName); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java index 1620779e57e..6a8e131ba07 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfigEntity.java @@ -58,7 +58,6 @@ public class BlueprintConfigEntity implements BlueprintConfiguration { @JoinColumn(name = "blueprint_name", referencedColumnName = "blueprint_name", nullable = false) private BlueprintEntity blueprint; - /** * Get the configuration type. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java index cddac9b3270..186738b0733 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintConfiguration.java @@ -36,13 +36,6 @@ public interface BlueprintConfiguration { */ String getType(); - /** - * Set the blueprint name. - * - * @param blueprintName blueprint name - */ - void setBlueprintName(String blueprintName); - /** * Get the blueprint name. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java index ea92a7be0e5..033cbd73dcf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintEntity.java @@ -57,7 +57,7 @@ public class BlueprintEntity { @Basic @Column(name = "security_descriptor_reference", nullable = true, insertable = true, updatable = true) private String securityDescriptorReference; - + @OneToMany(cascade = CascadeType.ALL, mappedBy = "blueprint") private Collection hostGroups = new ArrayList<>(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java index abc76a78144..c1848d1dc8c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintMpackConfigEntity.java @@ -96,14 +96,6 @@ public String getConfigData() { return configData; } - /** - * @param blueprintName the name of the blueprint - */ - @Override - public void setBlueprintName(String blueprintName) { - throw new UnsupportedOperationException(); - } - /** * @return the name of the blueprint */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java index a2bcf4e8c7a..d3822882b85 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/BlueprintServiceConfigEntity.java @@ -80,14 +80,6 @@ public String getType() { return type; } - /** - * @param blueprintName the blueprint name - */ - @Override - public void setBlueprintName(String blueprintName) { - throw new UnsupportedOperationException("BlueprintServiceEntity.setBlueprintName"); - } - /** * @return the blueprint name */ diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java index dc64712f4ff..fb2f6927aa7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostGroupConfigEntity.java @@ -60,7 +60,6 @@ public class HostGroupConfigEntity implements BlueprintConfiguration { @Lob private String configAttributes; - @ManyToOne @JoinColumns({ @JoinColumn(name = "hostgroup_name", referencedColumnName = "name", nullable = false), @@ -68,7 +67,6 @@ public class HostGroupConfigEntity implements BlueprintConfiguration { }) private HostGroupEntity hostGroup; - /** * Get the configuration type. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java index ba123de25b2..47000db783b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/NoSuchStackException.java @@ -25,4 +25,9 @@ public class NoSuchStackException extends Exception { public NoSuchStackException(String stackName, String stackVersion) { super(String.format("The requested stack doesn't exist. Name='%s' Version='%s'", stackName, stackVersion)); } + + public NoSuchStackException(String stackName, String stackVersion, Throwable cause) { + super(String.format("The requested stack doesn't exist. Name='%s' Version='%s'", stackName, stackVersion), cause); + } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 9f529725cb7..37840f6dfe3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -350,7 +350,7 @@ public void createAmbariServiceAndComponentResources(ClusterTopology topology, S serviceRequests.add(new ServiceRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, service, repositoryVersionId, null, credentialStoreEnabled, null)); - for (String component : topology.getBlueprint().getComponents(service)) { + for (String component : topology.getBlueprint().getComponentNames(service)) { String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component); componentRequests.add(new ServiceComponentRequest(clusterName, DEFAULT_SERVICE_GROUP_NAME, service, component, null, recoveryEnabled)); } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java index 6ed38f86f74..64d2ae34dda 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Blueprint.java @@ -78,6 +78,15 @@ public interface Blueprint { */ Collection getServices(); + /** + * Get the components that are included in the blueprint for the specified service. + * + * @param service service name + * + * @return collection of components for the service. Will not return null. + */ + Collection getComponents(String service); + /** * Get the components that are included in the blueprint for the specified service. * @@ -85,7 +94,8 @@ public interface Blueprint { * * @return collection of component names for the service. Will not return null. */ - Collection getComponents(String service); + @Deprecated + Collection getComponentNames(String service); /** * Get whether a component is enabled for auto start. @@ -117,8 +127,24 @@ public interface Blueprint { * * @return associated stack */ + @Deprecated Stack getStack(); + /** + * Get the stacks associated with the blueprint. + * + * @return associated stacks + */ + Collection getStacks(); + + + /** + * Get the mpacks associated with the blueprint. + * + * @return associated mpacks + */ + Collection getMpacks(); + /** * Get the host groups which contain components for the specified service. * @@ -170,4 +196,15 @@ public interface Blueprint { BlueprintEntity toEntity(); List getRepositorySettings(); + + /** + * @return a boolean indicating if all mpack referenced by the blueprints are resolved (installed on the system) + */ + boolean isAllMpacksResolved(); + + /** + * @return a collection of the names unresolved mpacks in {name}-{version} format. + */ + Collection getUnresolvedMpackNames(); + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java index 24b4785562c..48f41a62d31 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintFactory.java @@ -19,8 +19,10 @@ package org.apache.ambari.server.topology; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -37,7 +39,12 @@ import org.apache.ambari.server.orm.dao.BlueprintDAO; import org.apache.ambari.server.orm.entities.BlueprintEntity; import org.apache.ambari.server.stack.NoSuchStackException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; /** @@ -45,6 +52,8 @@ */ public class BlueprintFactory { + private static final Logger LOG = LoggerFactory.getLogger(BlueprintFactory.class); + // Blueprints protected static final String BLUEPRINT_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("Blueprints", "blueprint_name"); @@ -70,6 +79,10 @@ public class BlueprintFactory { protected static final String SETTINGS_PROPERTY_ID = "settings"; + protected static final String MPACK_INSTANCES_PROPERTY = "mpack_instances"; + protected static final String MPACK_INSTANCE_PROPERTY = "mpack_instance"; + protected static final String SERVICE_INSTANCE_PROPERTY = "service_instance"; + private static BlueprintDAO blueprintDAO; private ConfigurationFactory configFactory = new ConfigurationFactory(); @@ -105,26 +118,77 @@ public Blueprint createBlueprint(Map properties, SecurityConfigu throw new IllegalArgumentException("Blueprint name must be provided"); } - Stack stack = createStack(properties); + Stack stack; + Collection mpackInstances = createMpackInstances(properties); + if (mpackInstances.isEmpty()) { + stack = createStack(properties); + } + else { + stack = mpackInstances.iterator().next().getStack(); + } + Collection hostGroups = processHostGroups(name, stack, properties); Configuration configuration = configFactory.getConfiguration((Collection>) properties.get(CONFIGURATION_PROPERTY_ID)); - Setting setting = SettingFactory.getSetting((Collection>) properties.get(SETTINGS_PROPERTY_ID)); + Setting setting = SettingFactory.getSetting((Collection>) properties.get(SETTINGS_PROPERTY_ID)); + + if (!mpackInstances.isEmpty()) { + return new BlueprintImpl(name, hostGroups, mpackInstances, configuration, securityConfiguration, setting); + } + else { + // Legacy constructor for old blueprints without mpacks + return new BlueprintImpl(name, hostGroups, stack, configuration, securityConfiguration, setting); + } + } + + private Collection createMpackInstances(Map properties) throws NoSuchStackException { + if (properties.containsKey(MPACK_INSTANCES_PROPERTY)) { + ObjectMapper mapper = new ObjectMapper(); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + try { + String mpackInstancesJson = mapper.writeValueAsString(properties.get(MPACK_INSTANCES_PROPERTY)); + Collection mpacks = mapper.readValue(mpackInstancesJson, new TypeReference>(){}); + for (MpackInstance mpack: mpacks) { + tryResolveStack(mpack); + } + return mpacks; + } + catch (IOException ex) { + throw new RuntimeException("Unable to parse mpack instances for blueprint: " + + String.valueOf(properties.get(BLUEPRINT_NAME_PROPERTY_ID)), ex); + } + } else { + return Collections.emptyList(); + } + } - return new BlueprintImpl(name, hostGroups, stack, configuration, securityConfiguration, setting); + protected void tryResolveStack(MpackInstance mpack) { + try { + Stack stack = loadStack(mpack.getMpackName(), mpack.getMpackVersion()); + mpack.setStack(stack); + } + catch (NoSuchStackException ex) { + // This case can be normal if a blueprint had been sent in before the referenced mpack was installed + LOG.warn("Cannot resolve stack for mpack {}-{}. Is mpack installed?", mpack.getMpackName(), mpack.getMpackVersion()); + } } protected Stack createStack(Map properties) throws NoSuchStackException { String stackName = String.valueOf(properties.get(STACK_NAME_PROPERTY_ID)); String stackVersion = String.valueOf(properties.get(STACK_VERSION_PROPERTY_ID)); + return loadStack(stackName, stackVersion); + } + + protected Stack loadStack(String stackName, String stackVersion) throws NoSuchStackException { try { //todo: don't pass in controller return stackFactory.createStack(stackName, stackVersion, AmbariServer.getController()); } catch (ObjectNotFoundException e) { throw new NoSuchStackException(stackName, stackVersion); } catch (AmbariException e) { - //todo: - throw new RuntimeException("An error occurred parsing the stack information.", e); + // todo + throw new RuntimeException( + String.format("An error occurred parsing the stack information for %s-%s", stackName, stackVersion) , e); } } @@ -155,7 +219,7 @@ private Collection processHostGroups(String bpName, Stack stack, Map< Configuration configuration = configFactory.getConfiguration(configProps); String cardinality = String.valueOf(hostGroupProperties.get(HOST_GROUP_CARDINALITY_PROPERTY_ID)); - HostGroup group = new HostGroupImpl(hostGroupName, bpName, stack, components, configuration, cardinality); + HostGroup group = new HostGroupImpl(hostGroupName, bpName, Collections.singleton(stack), components, configuration, cardinality); hostGroups.add(group); } @@ -182,13 +246,12 @@ private Collection processHostGroupComponents(Stack stack, String gro groupName + "' is not valid for the specified stack"); } - String componentProvisionAction = componentProperties.get(COMPONENT_PROVISION_ACTION_PROPERTY_ID); - if (componentProvisionAction != null) { - //TODO, might want to add some validation here, to only accept value enum types, rwn - components.add(new Component(componentName, ProvisionAction.valueOf(componentProvisionAction))); - } else { - components.add(new Component(componentName)); - } + String mpackInstance = componentProperties.get(MPACK_INSTANCE_PROPERTY); + String serviceInstance = componentProperties.get(SERVICE_INSTANCE_PROPERTY); + //TODO, might want to add some validation here, to only accept value enum types, rwn + ProvisionAction provisionAction = componentProperties.containsKey(COMPONENT_PROVISION_ACTION_PROPERTY_ID) ? + ProvisionAction.valueOf(componentProperties.get(COMPONENT_PROVISION_ACTION_PROPERTY_ID)) : null; + components.add(new Component(componentName, mpackInstance, serviceInstance, provisionAction)); } return components; @@ -219,7 +282,7 @@ private Collection getAllStackComponents(Stack stack) { */ @Inject public static void init(BlueprintDAO dao) { - blueprintDAO = dao; + blueprintDAO = dao; } /** @@ -229,7 +292,7 @@ public static void init(BlueprintDAO dao) { * simulate various Stack or error conditions. */ interface StackFactory { - Stack createStack(String stackName, String stackVersion, AmbariManagementController managementController) throws AmbariException; + Stack createStack(String stackName, String stackVersion, AmbariManagementController managementController) throws AmbariException; } /** diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java index 1552a6cd062..24e5d3631de 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintImpl.java @@ -19,12 +19,17 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toList; + import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.apache.ambari.server.AmbariException; @@ -34,15 +39,21 @@ import org.apache.ambari.server.orm.entities.BlueprintConfigEntity; import org.apache.ambari.server.orm.entities.BlueprintConfiguration; import org.apache.ambari.server.orm.entities.BlueprintEntity; +import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity; +import org.apache.ambari.server.orm.entities.BlueprintServiceEntity; import org.apache.ambari.server.orm.entities.BlueprintSettingEntity; import org.apache.ambari.server.orm.entities.HostGroupComponentEntity; import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; -import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.stack.NoSuchStackException; import org.apache.ambari.server.state.ConfigHelper; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.base.Supplier; import com.google.gson.Gson; /** @@ -50,51 +61,73 @@ */ public class BlueprintImpl implements Blueprint { + private static final Logger LOG = LoggerFactory.getLogger(BlueprintImpl.class); + private String name; private Map hostGroups = new HashMap<>(); - private Stack stack; + private Collection mpacks = new ArrayList<>(); private Configuration configuration; private BlueprintValidator validator; private SecurityConfiguration security; private Setting setting; - private List repoSettings; + private List repoSettings = new ArrayList<>(); + private boolean allMpacksResolved = false; public BlueprintImpl(BlueprintEntity entity) throws NoSuchStackException { this.name = entity.getBlueprintName(); if (entity.getSecurityType() != null) { - this.security = new SecurityConfiguration(entity.getSecurityType(), entity.getSecurityDescriptorReference(), + this.security = new SecurityConfiguration(entity.getSecurityType(), + entity.getSecurityDescriptorReference(), null); } + mpacks.addAll(parseMpacks(entity)); // create config first because it is set as a parent on all host-group configs - processConfiguration(entity.getConfigurations()); + configuration = processConfiguration(entity.getConfigurations()); parseBlueprintHostGroups(entity); - configuration.setParentConfiguration(stack.getConfiguration(getServices())); + // TODO: how to handle multiple stacks correctly? +// configuration.setParentConfiguration(stack.getConfiguration(getServices())); validator = new BlueprintValidatorImpl(this); processSetting(entity.getSettings()); processRepoSettings(); } + /** + * Legacy constructor for pre-multi-mpack code. + * @param name blueprint name + * @param groups host groups + * @param stack stack + * @param configuration configuration + * @param security security config + * @param setting setting + */ + @Deprecated public BlueprintImpl(String name, Collection groups, Stack stack, Configuration configuration, - SecurityConfiguration security) { - this(name, groups, stack, configuration, security, null); + SecurityConfiguration security, Setting setting) { + this(name, groups, stackToMpacks(stack), configuration, security, setting); } - public BlueprintImpl(String name, Collection groups, Stack stack, Configuration configuration, - SecurityConfiguration security, Setting setting) { + private static Collection stackToMpacks(Stack stack) { + MpackInstance mpack = new MpackInstance(stack.getName(), stack.getVersion(), null, stack, new Configuration()); + return Collections.singleton(mpack); + } + + public BlueprintImpl(String name, Collection groups, Collection mpacks, + Configuration configuration, SecurityConfiguration security, Setting setting) { this.name = name; - this.stack = stack; + this.mpacks = mpacks; this.security = security; // caller should set host group configs for (HostGroup hostGroup : groups) { hostGroups.put(hostGroup.getName(), hostGroup); } + // TODO: handle configuration from multiple stacks properly // if the parent isn't set, the stack configuration is set as the parent this.configuration = configuration; - if (configuration.getParentConfiguration() == null) { - configuration.setParentConfiguration(stack.getConfiguration(getServices())); - } +// if (configuration.getParentConfiguration() == null) { +// configuration.setParentConfiguration(stack.getConfiguration(getServices())); +// } validator = new BlueprintValidatorImpl(this); this.setting = setting; } @@ -103,14 +136,6 @@ public String getName() { return name; } - public String getStackName() { - return stack.getName(); - } - - public String getStackVersion() { - return stack.getVersion(); - } - public SecurityConfiguration getSecurity() { return security; } @@ -152,15 +177,20 @@ public Collection getServices() { } @Override - public Collection getComponents(String service) { - Collection components = new HashSet<>(); + public Collection getComponents(String service) { + Collection components = new HashSet<>(); for (HostGroup group : getHostGroupsForService(service)) { components.addAll(group.getComponents(service)); } - return components; } + @Override + @Deprecated + public Collection getComponentNames(String service) { + return getComponents(service).stream().map(Component::getName).collect(toList()); + } + /** * Get whether the specified component in the service is enabled * for auto start. @@ -272,15 +302,25 @@ public boolean shouldSkipFailure() { return false; } + @Override + public Collection getMpacks() { + return mpacks; + } + + @Override + public Collection getStacks() { + return mpacks.stream().map(MpackInstance::getStack).filter(s -> null != s).collect(toList()); + } + @Override public Stack getStack() { - return stack; + return getStacks().iterator().next(); } /** * Get host groups which contain a component. * - * @param component component name + * @param component component name * * @return collection of host groups which contain the specified component */ @@ -319,7 +359,6 @@ public void validateTopology() throws InvalidTopologyException { } public BlueprintEntity toEntity() { - BlueprintEntity entity = new BlueprintEntity(); entity.setBlueprintName(name); if (security != null) { @@ -332,12 +371,25 @@ public BlueprintEntity toEntity() { } createHostGroupEntities(entity); - createBlueprintConfigEntities(entity); + Collection configEntities = toConfigEntities(getConfiguration(), () -> new BlueprintConfigEntity()); + configEntities.forEach(configEntity -> { + configEntity.setBlueprintEntity(entity); + configEntity.setBlueprintName(getName()); + }); + entity.setConfigurations(configEntities); createBlueprintSettingEntities(entity); - + createMpackInstanceEntities(entity); return entity; } + private void createMpackInstanceEntities(BlueprintEntity entity) { + mpacks.forEach(mpack -> { + BlueprintMpackInstanceEntity mpackEntity = mpack.toEntity(); + mpackEntity.setBlueprint(entity); + entity.getMpackInstances().add(mpackEntity); + }); + } + /** * Validate blueprint configuration. * @@ -349,21 +401,54 @@ public void validateRequiredProperties() throws InvalidTopologyException, GPLLic validator.validateRequiredProperties(); } - private void parseStack(StackEntity stackEntity) throws NoSuchStackException { + private Collection parseMpacks(BlueprintEntity blueprintEntity) throws NoSuchStackException { + Collection mpackInstances = new ArrayList<>(); + for (BlueprintMpackInstanceEntity mpack: blueprintEntity.getMpackInstances()) { + MpackInstance mpackInstance = new MpackInstance(); + mpackInstance.setMpackName(mpack.getMpackName()); + mpackInstance.setMpackVersion(mpack.getMpackVersion()); + mpackInstance.setUrl(mpack.getMpackUri()); + mpackInstance.setConfiguration(processConfiguration(mpack.getConfigurations())); + // TODO: come up with proper mpack -> stack resolution + tryParseStack(mpack.getMpackName(), mpack.getMpackVersion()).ifPresent( stack -> mpackInstance.setStack(stack) ); + for(BlueprintServiceEntity serviceEntity: mpack.getServiceInstances()) { + ServiceInstance serviceInstance = new ServiceInstance( + serviceEntity.getName(), + serviceEntity.getType(), + processConfiguration(serviceEntity.getConfigurations())); + mpackInstance.addServiceInstance(serviceInstance); + } + mpackInstances.add(mpackInstance); + } + return mpackInstances; + } + + private Stack parseStack(String stackName, String stackVersion) throws NoSuchStackException { try { //todo: don't pass in controller - stack = new Stack(stackEntity.getStackName(), stackEntity.getStackVersion(), AmbariServer.getController()); + return new Stack(stackName, stackVersion, AmbariServer.getController()); } catch (StackAccessException e) { - throw new NoSuchStackException(stackEntity.getStackName(), stackEntity.getStackVersion()); + throw new NoSuchStackException(stackName, stackVersion, e); } catch (AmbariException e) { //todo: throw new RuntimeException("An error occurred parsing the stack information.", e); } } + private Optional tryParseStack(String stackName, String stackVersion) { + try { + return Optional.of(parseStack(stackName, stackVersion)); + } + catch (Exception ex) { + LOG.warn("Cannot parse stack {}-{}. Exception: {}/{}", stackName, stackVersion, ex.getClass().getName(), + ex.getMessage()); + return Optional.empty(); + } + } + private Map parseBlueprintHostGroups(BlueprintEntity entity) { for (HostGroupEntity hostGroupEntity : entity.getHostGroups()) { - HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), stack); + HostGroupImpl hostGroup = new HostGroupImpl(hostGroupEntity, getName(), getStacks()); // set the bp configuration as the host group config parent hostGroup.getConfiguration().setParentConfiguration(configuration); hostGroups.put(hostGroupEntity.getName(), hostGroup); @@ -374,9 +459,9 @@ private Map parseBlueprintHostGroups(BlueprintEntity entity) /** * Process blueprint configurations. */ - private void processConfiguration(Collection configs) { + private Configuration processConfiguration(Collection configs) { // not setting stack configuration as parent until after host groups are parsed in constructor - configuration = new Configuration(parseConfigurations(configs), + return new Configuration(parseConfigurations(configs), parseAttributes(configs), null); } @@ -396,8 +481,7 @@ private void processSetting(Collection blueprintSetting) * * @return map of config type to map of properties */ - private Map> parseConfigurations(Collection configs) { - + private Map> parseConfigurations(Collection configs) { Map> properties = new HashMap<>(); Gson gson = new Gson(); for (BlueprintConfiguration config : configs) { @@ -414,7 +498,7 @@ private Map> parseConfigurations(Collection>> parseSetting(Collection blueprintSetting) { + private Map>> parseSetting(Collection blueprintSetting) { Map>> properties = new HashMap<>(); Gson gson = new Gson(); @@ -433,13 +517,13 @@ private Map>> parseSetting(Collection>> parseAttributes(Collection configs) { + private Map>> parseAttributes(Collection configs) { Map>> mapAttributes = new HashMap<>(); if (configs != null) { Gson gson = new Gson(); - for (BlueprintConfigEntity config : configs) { + for (BlueprintConfiguration config : configs) { Map> typeAttrs = gson.>>fromJson(config.getConfigAttributes(), Map.class); if (typeAttrs != null && !typeAttrs.isEmpty()) { @@ -465,50 +549,19 @@ private void createHostGroupEntities(BlueprintEntity blueprintEntity) { hostGroupEntity.setBlueprintName(getName()); hostGroupEntity.setCardinality(group.getCardinality()); - createHostGroupConfigEntities(hostGroupEntity, group.getConfiguration()); + Collection configEntities = toConfigEntities(group.getConfiguration(), () -> new HostGroupConfigEntity()); + configEntities.forEach(configEntity -> { + configEntity.setBlueprintName(getName()); + configEntity.setHostGroupEntity(hostGroupEntity); + configEntity.setHostGroupName(group.getName()); + }); + hostGroupEntity.setConfigurations(configEntities); createComponentEntities(hostGroupEntity, group.getComponents()); } blueprintEntity.setHostGroups(entities); } - /** - * Populate host group configurations. - */ - private void createHostGroupConfigEntities(HostGroupEntity hostGroup, Configuration groupConfiguration) { - Gson jsonSerializer = new Gson(); - Map configEntityMap = new HashMap<>(); - for (Map.Entry> propEntry : groupConfiguration.getProperties().entrySet()) { - String type = propEntry.getKey(); - Map properties = propEntry.getValue(); - - HostGroupConfigEntity configEntity = new HostGroupConfigEntity(); - configEntityMap.put(type, configEntity); - configEntity.setBlueprintName(getName()); - configEntity.setHostGroupEntity(hostGroup); - configEntity.setHostGroupName(hostGroup.getName()); - configEntity.setType(type); - configEntity.setConfigData(jsonSerializer.toJson(properties)); - } - - for (Map.Entry>> attributesEntry : groupConfiguration.getAttributes().entrySet()) { - String type = attributesEntry.getKey(); - Map> attributes = attributesEntry.getValue(); - - HostGroupConfigEntity entity = configEntityMap.get(type); - if (entity == null) { - entity = new HostGroupConfigEntity(); - configEntityMap.put(type, entity); - entity.setBlueprintName(getName()); - entity.setHostGroupEntity(hostGroup); - entity.setHostGroupName(hostGroup.getName()); - entity.setType(type); - } - entity.setConfigAttributes(jsonSerializer.toJson(attributes)); - } - hostGroup.setConfigurations(configEntityMap.values()); - } - /** * Create component entities and add to parent host group. */ @@ -525,6 +578,17 @@ private void createComponentEntities(HostGroupEntity group, Collection mpackNameAndVersion = + Splitter.on('-').split(component.getMpackInstance()).iterator(); + componentEntity.setMpackName(mpackNameAndVersion.next()); + componentEntity.setMpackVersion(mpackNameAndVersion.next()); + } // add provision action (if specified) to entity type // otherwise, just leave this column null (provision_action) @@ -537,39 +601,62 @@ private void createComponentEntities(HostGroupEntity group, Collection the config entity's type + * @return a collection of configuration entities */ - private void createBlueprintConfigEntities(BlueprintEntity blueprintEntity) { + static Collection toConfigEntities(Configuration configuration, Supplier entityCreator) { Gson jsonSerializer = new Gson(); - Configuration config = getConfiguration(); - Map configEntityMap = new HashMap<>(); - for (Map.Entry> propEntry : config.getProperties().entrySet()) { + Map configEntityMap = new HashMap<>(); + for (Map.Entry> propEntry : configuration.getProperties().entrySet()) { String type = propEntry.getKey(); Map properties = propEntry.getValue(); - BlueprintConfigEntity configEntity = new BlueprintConfigEntity(); + E configEntity = entityCreator.get(); configEntityMap.put(type, configEntity); - configEntity.setBlueprintName(getName()); - configEntity.setBlueprintEntity(blueprintEntity); configEntity.setType(type); configEntity.setConfigData(jsonSerializer.toJson(properties)); } - for (Map.Entry>> attributesEntry : config.getAttributes().entrySet()) { + for (Map.Entry>> attributesEntry : configuration.getAttributes().entrySet()) { String type = attributesEntry.getKey(); Map> attributes = attributesEntry.getValue(); - BlueprintConfigEntity entity = configEntityMap.get(type); + E entity = configEntityMap.get(type); if (entity == null) { - entity = new BlueprintConfigEntity(); + entity = entityCreator.get(); configEntityMap.put(type, entity); - entity.setBlueprintName(getName()); - entity.setBlueprintEntity(blueprintEntity); entity.setType(type); } entity.setConfigAttributes(jsonSerializer.toJson(attributes)); } - blueprintEntity.setConfigurations(configEntityMap.values()); + return configEntityMap.values(); + } + + /** + * Converts a collection of configuration entities into a {@link Configuration} + * @param configEntities the persisted configuration as entitiy collection + * @return the configuration + */ + static Configuration fromConfigEntities(Collection configEntities) { + Gson jsonSerializer = new Gson(); + Configuration configuration = new Configuration(); + + for (BlueprintConfiguration configEntity: configEntities) { + String type = configEntity.getType(); + Map configData = jsonSerializer.fromJson(configEntity.getConfigData(), Map.class); + Map> configAttributes = jsonSerializer.fromJson(configEntity.getConfigAttributes(), Map.class); + if (null != configData) { + configuration.getProperties().put(type, configData); + } + if (null != configAttributes) { + configuration.getAttributes().put(type, configAttributes); + } + } + + return configuration; } /** @@ -602,9 +689,12 @@ public boolean isValidConfigType(String configType) { if (ConfigHelper.CLUSTER_ENV.equals(configType) || "global".equals(configType)) { return true; } - String service = getStack().getServiceForConfigType(configType); - if (getServices().contains(service)) { + + Collection services = getStacks().stream().map(stack -> stack.getServiceForConfigType(configType)).collect(toList()); + for (String service: services) { + if (getServices().contains(service)) { return true; + } } return false; } @@ -636,4 +726,20 @@ private RepositorySetting parseRepositorySetting(Map setting){ public List getRepositorySettings(){ return repoSettings; } + + /** + * {@inheritDoc} + */ + @Override + public boolean isAllMpacksResolved() { + return !mpacks.stream().filter(mpack -> mpack.getStack() == null).findAny().isPresent(); + } + + /** + * {@inheritDoc} + */ + public Collection getUnresolvedMpackNames() { + return mpacks.stream().filter(mpack -> mpack.getStack() == null).map(MpackInstance::getMpackNameAndVersion).collect(toList()); + } + } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java index fbd0e4bc134..1e6213edc35 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java @@ -36,6 +36,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; import com.google.inject.Inject; /** @@ -63,6 +64,13 @@ public BlueprintValidatorImpl(Blueprint blueprint) { @Override public void validateTopology() throws InvalidTopologyException { LOGGER.info("Validating topology for blueprint: [{}]", blueprint.getName()); + + if (!blueprint.isAllMpacksResolved()) { + LOGGER.warn("The following macks are not resolved: [{}] Skipping topology validation.", + Joiner.on(", ").join(blueprint.getUnresolvedMpackNames())); + return; + } + Collection hostGroups = blueprint.getHostGroups().values(); Map>> missingDependencies = new HashMap<>(); @@ -81,10 +89,10 @@ public void validateTopology() throws InvalidTopologyException { Cardinality cardinality = stack.getCardinality(component); AutoDeployInfo autoDeploy = stack.getAutoDeployInfo(component); if (cardinality.isAll()) { - cardinalityFailures.addAll(verifyComponentInAllHostGroups(component, autoDeploy)); + cardinalityFailures.addAll(verifyComponentInAllHostGroups(new Component(component), autoDeploy)); } else { cardinalityFailures.addAll(verifyComponentCardinalityCount( - component, cardinality, autoDeploy)); + new Component(component), cardinality, autoDeploy)); } } } @@ -178,27 +186,33 @@ public void validateRequiredProperties() throws InvalidTopologyException, GPLLic } } - if (component.equals("HIVE_METASTORE")) { - Map hiveEnvConfig = clusterConfigurations.get("hive-env"); - if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null - && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") - && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 - && stack.getName().equalsIgnoreCase("HDP")) { - throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + - "and repo version 2.3.2+!"); + if (blueprint.isAllMpacksResolved()) { + if (component.equals("HIVE_METASTORE")) { + Map hiveEnvConfig = clusterConfigurations.get("hive-env"); + if (hiveEnvConfig != null && !hiveEnvConfig.isEmpty() && hiveEnvConfig.get("hive_database") != null + && hiveEnvConfig.get("hive_database").equals("Existing SQL Anywhere Database") + && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 + && stack.getName().equalsIgnoreCase("HDP")) { + throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + + "and repo version 2.3.2+!"); + } } - } - if (component.equals("OOZIE_SERVER")) { - Map oozieEnvConfig = clusterConfigurations.get("oozie-env"); - if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") != null - && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") - && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 - && stack.getName().equalsIgnoreCase("HDP")) { - throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + - "and repo version 2.3.2+!"); + if (component.equals("OOZIE_SERVER")) { + Map oozieEnvConfig = clusterConfigurations.get("oozie-env"); + if (oozieEnvConfig != null && !oozieEnvConfig.isEmpty() && oozieEnvConfig.get("oozie_database") != null + && oozieEnvConfig.get("oozie_database").equals("Existing SQL Anywhere Database") + && VersionUtils.compareVersions(stack.getVersion(), "2.3.0.0") < 0 + && stack.getName().equalsIgnoreCase("HDP")) { + throw new InvalidTopologyException("Incorrect configuration: SQL Anywhere db is available only for stack HDP-2.3+ " + + "and repo version 2.3.2+!"); + } } } + else { + LOGGER.warn("The following macks are not resolved: [{}] Skipping validation of HIVE_METASTORE and OOZIE_SERVER properties.", + Joiner.on(", ").join(blueprint.getUnresolvedMpackNames())); + } } } } @@ -212,10 +226,10 @@ public void validateRequiredProperties() throws InvalidTopologyException, GPLLic * * @return collection of missing component information */ - private Collection verifyComponentInAllHostGroups(String component, AutoDeployInfo autoDeploy) { + private Collection verifyComponentInAllHostGroups(Component component, AutoDeployInfo autoDeploy) { Collection cardinalityFailures = new HashSet<>(); - int actualCount = blueprint.getHostGroupsForComponent(component).size(); + int actualCount = blueprint.getHostGroupsForComponent(component.getName()).size(); Map hostGroups = blueprint.getHostGroups(); if (actualCount != hostGroups.size()) { if (autoDeploy != null && autoDeploy.isEnabled()) { @@ -233,10 +247,10 @@ private Map> validateHostGroup(HostGroup grou LOGGER.info("Validating hostgroup: {}", group.getName()); Map> missingDependencies = new HashMap<>(); - for (String component : new HashSet<>(group.getComponentNames())) { + for (Component component : new HashSet<>(group.getComponents())) { LOGGER.debug("Processing component: {}", component); - for (DependencyInfo dependency : stack.getDependenciesForComponent(component)) { + for (DependencyInfo dependency : stack.getDependenciesForComponent(component.getName())) { LOGGER.debug("Processing dependency [{}] for component [{}]", dependency.getName(), component); String conditionalService = stack.getConditionalServiceForDependency(dependency); @@ -274,21 +288,21 @@ private Map> validateHostGroup(HostGroup grou } if (dependencyScope.equals("cluster")) { Collection missingDependencyInfo = verifyComponentCardinalityCount( - componentName, new Cardinality("1+"), autoDeployInfo); + new Component(componentName), new Cardinality("1+"), autoDeployInfo); resolved = missingDependencyInfo.isEmpty(); } else if (dependencyScope.equals("host")) { if (group.getComponentNames().contains(componentName) || (autoDeployInfo != null && autoDeployInfo.isEnabled())) { resolved = true; - group.addComponent(componentName); + group.addComponent(new Component(componentName)); } } if (! resolved) { - Collection missingCompDependencies = missingDependencies.get(component); + Collection missingCompDependencies = missingDependencies.get(component.getName()); if (missingCompDependencies == null) { missingCompDependencies = new HashSet<>(); - missingDependencies.put(component, missingCompDependencies); + missingDependencies.put(component.getName(), missingCompDependencies); } missingCompDependencies.add(dependency); } @@ -307,7 +321,7 @@ private Map> validateHostGroup(HostGroup grou * * @return collection of missing component information */ - public Collection verifyComponentCardinalityCount(String component, + public Collection verifyComponentCardinalityCount(Component component, Cardinality cardinality, AutoDeployInfo autoDeploy) { @@ -315,15 +329,15 @@ public Collection verifyComponentCardinalityCount(String component, Collection cardinalityFailures = new HashSet<>(); //todo: don't hard code this HA logic here if (ClusterTopologyImpl.isNameNodeHAEnabled(configProperties) && - (component.equals("SECONDARY_NAMENODE"))) { + (component.getName().equals("SECONDARY_NAMENODE"))) { // override the cardinality for this component in an HA deployment, // since the SECONDARY_NAMENODE should not be started in this scenario cardinality = new Cardinality("0"); } - int actualCount = blueprint.getHostGroupsForComponent(component).size(); + int actualCount = blueprint.getHostGroupsForComponent(component.getName()).size(); if (! cardinality.isValidCount(actualCount)) { - boolean validated = ! isDependencyManaged(stack, component, configProperties); + boolean validated = ! isDependencyManaged(stack, component.getName(), configProperties); if (! validated && autoDeploy != null && autoDeploy.isEnabled() && cardinality.supportsAutoDeploy()) { String coLocateName = autoDeploy.getCoLocate(); if (coLocateName != null && ! coLocateName.isEmpty()) { diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 740dd91e447..dfe5dd01c55 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -237,11 +237,11 @@ private Map> createServiceComponentMap(Blueprint blueprint) if(services != null) { for (String service : services) { - Collection components = blueprint.getComponents(service); + Collection components = blueprint.getComponentNames(service); serviceComponents.put(service, (components == null) ? Collections.emptySet() - : new HashSet<>(blueprint.getComponents(service))); + : new HashSet<>(blueprint.getComponentNames(service))); } } @@ -281,7 +281,7 @@ private boolean propertyHasCustomValue(Map clusterConfigProperti private Map createComponentHostMap(Blueprint blueprint) { Map componentHostsMap = new HashMap<>(); for (String service : blueprint.getServices()) { - Collection components = blueprint.getComponents(service); + Collection components = blueprint.getComponentNames(service); for (String component : components) { Collection componentHost = clusterTopology.getHostAssignmentsForComponent(component); // retrieve corresponding clusterInfoKey for component using StageUtils diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java index 03620ad4ed8..a62d341d3cc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Component.java @@ -19,21 +19,31 @@ package org.apache.ambari.server.topology; +import java.util.Objects; + +import javax.annotation.Nullable; + import org.apache.ambari.server.controller.internal.ProvisionAction; public class Component { private final String name; + @Nullable + private final String mpackInstance; + @Nullable + private final String serviceInstance; private final ProvisionAction provisionAction; + @Deprecated public Component(String name) { - this(name, null); + this(name, null, null, null); } - - public Component(String name, ProvisionAction provisionAction) { + public Component(String name, String mpackInstance, String serviceInstance, ProvisionAction provisionAction) { this.name = name; + this.mpackInstance = mpackInstance; + this.serviceInstance = serviceInstance; this.provisionAction = provisionAction; } @@ -46,6 +56,21 @@ public String getName() { return this.name; } + /** + * @return the mpack associated with this component (can be {@code null} if component -> mpack mapping is unambigous) + */ + public String getMpackInstance() { + return mpackInstance; + } + + /** + * @return the service instance this component belongs to. Can be {@null} if component does not belong to a service + * instance (there is a single service of the component's service type) + */ + public String getServiceInstance() { + return serviceInstance; + } + /** * Gets the provision action associated with this component. * @@ -56,4 +81,30 @@ public ProvisionAction getProvisionAction() { return this.provisionAction; } + @Override + public String toString() { + return com.google.common.base.Objects.toStringHelper(this) + .add("name", name) + .add("mpackInstance", mpackInstance) + .add("serviceInstance", serviceInstance) + .add("provisionAction", provisionAction) + .toString(); + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Component component = (Component) o; + return Objects.equals(name, component.name) && + Objects.equals(mpackInstance, component.mpackInstance) && + Objects.equals(serviceInstance, component.serviceInstance) && + provisionAction == component.provisionAction; + } + + @Override + public int hashCode() { + return Objects.hash(name, mpackInstance, serviceInstance, provisionAction); + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java new file mode 100644 index 00000000000..b71b12c0bf8 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configurable.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import static org.apache.ambari.server.topology.BlueprintFactory.PROPERTIES_ATTRIBUTES_PROPERTY_ID; +import static org.apache.ambari.server.topology.BlueprintFactory.PROPERTIES_PROPERTY_ID; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; + + + +public interface Configurable { + + @JsonIgnore + void setConfiguration(Configuration configuration); + @JsonIgnore + Configuration getConfiguration(); + + @JsonProperty("configurations") + default void setConfigs(Collection> configs) { + Configuration configuration; + if (!configs.isEmpty() && configs.iterator().next().keySet().iterator().next().contains("/")) { + // Configuration has keys with slashes like "zk.cfg/properties/dataDir" means it is coming through + // the resource framework and must be parsed with configuration factories + configuration = new ConfigurationFactory().getConfiguration((Collection>)configs); + } + else { + // If the configuration does not have keys with slashes it means it is coming from plain JSON and needs to be + // parsed accordingly. + Map> allProperties = new HashMap<>(); + Map>> allAttributes = new HashMap<>(); + configs.forEach( item -> { + String configName = item.keySet().iterator().next(); + Map configData = (Map) item.get(configName); + if (configData.containsKey(PROPERTIES_PROPERTY_ID)) { + Map properties = (Map)configData.get(PROPERTIES_PROPERTY_ID); + allProperties.put(configName, properties); + } + if (configData.containsKey(PROPERTIES_ATTRIBUTES_PROPERTY_ID)) { + Map> attributes = + (Map>)configData.get(PROPERTIES_ATTRIBUTES_PROPERTY_ID); + allAttributes.put(configName, attributes); + } + }); + configuration = new Configuration(allProperties, allAttributes); + } + setConfiguration(configuration); + } + + @JsonProperty("configurations") + default Collection>> getConfigs() { + Configuration configuration = getConfiguration(); + Collection>> configurations = new ArrayList<>(); + Set allConfigTypes = Sets.union(configuration.getProperties().keySet(), configuration.getAttributes().keySet()); + for (String configType: allConfigTypes) { + Map> configData = new HashMap<>(); + if (configuration.getProperties().containsKey(configType)) { + configData.put(PROPERTIES_PROPERTY_ID, configuration.getProperties().get(configType)); + } + if (configuration.getAttributes().containsKey(configType)) { + configData.put(PROPERTIES_ATTRIBUTES_PROPERTY_ID, configuration.getAttributes().get(configType)); + } + configurations.add(ImmutableMap.of(configType, configData)); + } + return configurations; + } + +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java index 28b62bcd9de..8ffbdae771e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Configuration.java @@ -43,6 +43,12 @@ public class Configuration { */ private Configuration parentConfiguration; + + public Configuration() { + properties = new HashMap<>(); + attributes = new HashMap<>(); + } + /** * Constructor. * diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java index 3e80247c042..43ce5b92a93 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroup.java @@ -62,11 +62,13 @@ public interface HostGroup { */ Collection getComponents(); + /** * Get all of the host group component names * * @return collection of component names as String */ + @Deprecated Collection getComponentNames(); /** @@ -79,35 +81,38 @@ public interface HostGroup { * @return collection of component names as String that are associated with * the specified provision action */ + @Deprecated Collection getComponentNames(ProvisionAction provisionAction); /** - * Get the host group components which belong to the specified service. + * Get the names components for the specified service which are associated with the host group. * * @param service service name * - * @return collection of component names for the specified service; will not return null + * @return set of component names */ - Collection getComponents(String service); + @Deprecated + public Collection getComponentNames(String service); /** - * Add a component to the host group. + * Get the host group components which belong to the specified service. * - * @param component name of the component to add + * @param service service instance name or service name. First, services looked up + * by instance name. If no appropriate service instance is found, services are looked + * up by type * - * @return true if the component didn't already exist + * @return collection of component names for the specified service; will not return null */ - boolean addComponent(String component); + Collection getComponents(String service); + + /** - * Add a component to the host group, with the specified name - * and provision action. - * - * @param component component name - * @param provisionAction provision action for this component + * Add a component to the host group + * @param component * @return */ - boolean addComponent(String component, ProvisionAction provisionAction); + boolean addComponent(Component component); /** * Determine if the host group contains a master component. @@ -137,8 +142,16 @@ public interface HostGroup { * * @return associated stack */ + @Deprecated Stack getStack(); + /** + * Get the stack associated with the host group. + * + * @return associated stacks + */ + Collection getStacks(); + /** * Get the cardinality value that was specified for the host group. * This is simply meta-data for the stack that a deployer can use diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java index 9aeadd15361..88ad1627ca8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupImpl.java @@ -19,12 +19,19 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import org.apache.ambari.server.controller.internal.ProvisionAction; import org.apache.ambari.server.controller.internal.Stack; @@ -32,6 +39,10 @@ import org.apache.ambari.server.orm.entities.HostGroupConfigEntity; import org.apache.ambari.server.orm.entities.HostGroupEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; import com.google.gson.Gson; /** @@ -39,6 +50,8 @@ */ public class HostGroupImpl implements HostGroup { + private static final Logger LOG = LoggerFactory.getLogger(HostGroupImpl.class); + /** * host group name */ @@ -52,12 +65,13 @@ public class HostGroupImpl implements HostGroup { /** * components contained in the host group */ - private Map components = new HashMap<>(); + private final List components = new ArrayList<>(); /** * map of service to components for the host group */ - private Map> componentsForService = new HashMap<>(); + // TODO: in blueprint 3.0 this should be per service instance + private Map> componentsForService = new HashMap<>(); /** * configuration @@ -66,28 +80,29 @@ public class HostGroupImpl implements HostGroup { private boolean containsMasterComponent = false; - private Stack stack; + private Map stackMap; private String cardinality = "NOT SPECIFIED"; - public HostGroupImpl(HostGroupEntity entity, String blueprintName, Stack stack) { + public HostGroupImpl(HostGroupEntity entity, String blueprintName, Collection stacks) { this.name = entity.getName(); this.cardinality = entity.getCardinality(); this.blueprintName = blueprintName; - this.stack = stack; - + this.stackMap = stacks.stream().collect(Collectors.toMap(stack -> stack.getName() + "-" + stack.getVersion(), stack -> stack)); parseComponents(entity); parseConfigurations(entity); } - public HostGroupImpl(String name, String bpName, Stack stack, Collection components, Configuration configuration, String cardinality) { + public HostGroupImpl(String name, String bpName, Collection stacks, Collection components, Configuration configuration, String cardinality) { this.name = name; this.blueprintName = bpName; - this.stack = stack; + this.stackMap = stacks.stream().collect(Collectors.toMap(stack -> stack.getName() + "-" + stack.getVersion(), stack -> stack)); // process each component - for (Component component : components) { - addComponent(component.getName(), component.getProvisionAction()); + for (Component component: components) { + if (!addComponent(component)) { + throw new IllegalArgumentException("Ambiguous component or can't determine stack for: " + component); + } } this.configuration = configuration; @@ -115,28 +130,27 @@ public static String formatAbsoluteName(String bpName, String hgName) { @Override public Collection getComponents() { - return components.values(); + return components; } @Override + @Deprecated public Collection getComponentNames() { - return components.keySet(); + return components.stream().map(Component::getName).collect(toList()); } @Override + @Deprecated public Collection getComponentNames(ProvisionAction provisionAction) { Set setOfComponentNames = new HashSet<>(); - for (String componentName : components.keySet()) { - Component component = components.get(componentName); + for (Component component : components) { if ( (component.getProvisionAction() != null) && (component.getProvisionAction() == provisionAction) ) { - setOfComponentNames.add(componentName); + setOfComponentNames.add(component.getName()); } } - return setOfComponentNames; } - /** * Get the services which are deployed to this host group. * @@ -148,51 +162,82 @@ public Collection getServices() { } /** - * Add a component to the host group. - * - * @param component component to add - * - * @return true if component was added; false if component already existed + * Adds a component to the host group. The component is successfully added if it is not a duplicate or ambiguous (as of + * Ambari 3.1 multiple components of the same type can exist in a hostgroup. However, they have to come from different + * management packs or belong to different service instances) + * @param component the component to add + * @return a boolean to indicate if addition was successful */ - @Override - public boolean addComponent(String component) { - return this.addComponent(component, null); + public boolean addComponent(Component component) { + // Exclude ambiguous component definitions + boolean ambigous = components.stream().filter(c -> { + if (c.getName().equals(component.getName())) { // found another component with the same name + if (c.getMpackInstance() == null || component.getMpackInstance() == null) { + return true; // if either of them has no mpack instance defined it is ambiguous + } + if (c.getMpackInstance().equals(component.getMpackInstance())) { + // both components are in the same mpack, and one of them does not declare a service instance or + // both declare the same service instance --> ambiguous + return c.getServiceInstance() != null && component.getServiceInstance() != null && + c.getServiceInstance().equals(component.getServiceInstance()); + } + else { + return false; // different mpacks --> no ambiguity + } + } + else { + return false; // different name --> no ambiguity + } + }).findAny().isPresent(); + if (ambigous) { + return false; + } + addComponent(component, getStackForComponent(component)); + return true; } - /** - * Add a component with the specified provision action to the - * host group. - * - * @param component component name - * @param provisionAction provision action for this component - * - * @return true if component was added; false if component already existed - */ - public boolean addComponent(String component, ProvisionAction provisionAction) { - boolean added; - if (!components.containsKey(component)) { - components.put(component, new Component(component, provisionAction)); - added = true; - } else { - added = false; + private Optional getStackForComponent(Component component) { + // Look for the stack of this component + if (component.getMpackInstance() == null) { + // Component does not declare its stack. Let's find it. + Collection candidateStacks = + stackMap.values().stream().filter(stack -> stack.getServiceForComponent(component.getName()) != null).collect(toList()); + switch (candidateStacks.size()) { + case 0: + // no stack (no service) for this component + LOG.warn("No stack/service found for component: {}", component); + return Optional.empty(); + case 1: + return Optional.of(candidateStacks.iterator().next()); + default: + LOG.warn("Ambiguous stack resolution for component: {}, stacks: {}", component, candidateStacks); + return Optional.empty(); + } } - - if (stack.isMasterComponent(component)) { - containsMasterComponent = true; + else { + // TODO: refine this logic + Stack stack = stackMap.get(component.getMpackInstance()); + if (null == stack) { + LOG.warn("Component declared an invalid stack: {}", component); + } + return Optional.ofNullable(stack); } - if (added) { - String service = stack.getServiceForComponent(component); - if (service != null) { - // an example of a component without a service in the stack is AMBARI_SERVER - Set serviceComponents = componentsForService.get(service); - if (serviceComponents == null) { - serviceComponents = new HashSet<>(); - componentsForService.put(service, serviceComponents); - } - serviceComponents.add(component); + } + + private void addComponent(Component component, Optional stack) { + if (stack.isPresent()) { + String serviceName = stack.get().getServiceForComponent(component.getName()); + if (!componentsForService.containsKey(serviceName)) { + componentsForService.put(serviceName, Sets.newHashSet(component)); + } + else { + componentsForService.get(serviceName).add(component); + } + if (stack.get().isMasterComponent(component.getName())) { + containsMasterComponent = true; } } - return added; + components.add(component); } /** @@ -200,15 +245,31 @@ public boolean addComponent(String component, ProvisionAction provisionAction) { * * @param service service name * - * @return set of component names + * @return set of components */ @Override - public Collection getComponents(String service) { + public Collection getComponents(String service) { return componentsForService.containsKey(service) ? new HashSet<>(componentsForService.get(service)) : Collections.emptySet(); } + /** + * Get the names components for the specified service which are associated with the host group. + * + * @param service service name + * + * @return set of component names + */ + @Override + @Deprecated + public Collection getComponentNames(String service) { + return componentsForService.containsKey(service) ? + componentsForService.get(service).stream().map(Component::getName).collect(toSet()) : + Collections.emptySet(); + } + + /** * Get this host groups configuration. * @@ -216,7 +277,6 @@ public Collection getComponents(String service) { */ @Override public Configuration getConfiguration() { - return configuration; } @@ -236,8 +296,14 @@ public boolean containsMasterComponent() { } @Override + public Collection getStacks() { + return stackMap.values(); + } + + @Override + @Deprecated public Stack getStack() { - return stack; + return getStacks().iterator().next(); } @Override @@ -250,13 +316,12 @@ public String getCardinality() { */ private void parseComponents(HostGroupEntity entity) { for (HostGroupComponentEntity componentEntity : entity.getComponents() ) { - if (componentEntity.getProvisionAction() != null) { - addComponent(componentEntity.getName(), ProvisionAction.valueOf(componentEntity.getProvisionAction())); - } else { - addComponent(componentEntity.getName()); - } - - + Component component = new Component( + componentEntity.getName(), + componentEntity.getMpackName(), + componentEntity.getServiceName(), + null == componentEntity.getProvisionAction() ? null : ProvisionAction.valueOf(componentEntity.getProvisionAction())); + addComponent(component); } } @@ -270,13 +335,12 @@ private void parseConfigurations(HostGroupEntity entity) { for (HostGroupConfigEntity configEntity : entity.getConfigurations()) { String type = configEntity.getType(); Map typeProperties = config.get(type); - if ( typeProperties == null) { + if (typeProperties == null) { typeProperties = new HashMap<>(); config.put(type, typeProperties); } Map propertyMap = jsonSerializer.>fromJson( configEntity.getConfigData(), Map.class); - if (propertyMap != null) { typeProperties.putAll(propertyMap); } @@ -285,6 +349,7 @@ private void parseConfigurations(HostGroupEntity entity) { Map>> attributes = new HashMap<>(); configuration = new Configuration(config, attributes); } + public String toString(){ return name; } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java index 4648412925e..b00faad08b1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/HostGroupInfo.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -76,6 +77,8 @@ public class HostGroupInfo { */ Predicate predicate; + private Collection managementPackMappings = new ArrayList<>(); + /** * Constructor @@ -242,4 +245,11 @@ public void removeHost(String hostname) { } } + public Collection getManagementPackMappings() { + return managementPackMappings; + } + + public void setManagementPackMappings(Collection managementPackMappings) { + this.managementPackMappings = managementPackMappings; + } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java new file mode 100644 index 00000000000..334991b8995 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ManagementPackMapping.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Iterator; + +import com.google.common.base.Splitter; + +/** + * Represents a component -> mpack mapping in the multi-everything cluster topology template. + */ +public class ManagementPackMapping { + private String componentName; + private String mpackName; + private String mpackVersion; + + public ManagementPackMapping(String componentName, String mpackNameAndVersion) { + this.componentName = componentName; + checkArgument(mpackNameAndVersion.contains("-"), + "Management pack must be specified in {name}-{version} format. Received %s", mpackNameAndVersion); + Iterator mpackNameIterator = Splitter.on('-').split(mpackNameAndVersion).iterator(); + this.mpackName = mpackNameIterator.next(); + this.mpackVersion = mpackNameIterator.next(); + } + + public String getComponentName() { + return componentName; + } + + public void setComponentName(String componentName) { + this.componentName = componentName; + } + + public String getMpackName() { + return mpackName; + } + + public void setMpackName(String mpackName) { + this.mpackName = mpackName; + } + + public String getMpackVersion() { + return mpackVersion; + } + + public void setMpackVersion(String mpackVersion) { + this.mpackVersion = mpackVersion; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java new file mode 100644 index 00000000000..cde01e0565f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/MpackInstance.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.orm.entities.BlueprintMpackConfigEntity; +import org.apache.ambari.server.orm.entities.BlueprintMpackInstanceEntity; +import org.apache.ambari.server.orm.entities.BlueprintServiceConfigEntity; +import org.apache.ambari.server.orm.entities.BlueprintServiceEntity; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class MpackInstance implements Configurable { + @JsonProperty("name") + private String mpackName; + @JsonProperty("version") + private String mpackVersion; + @JsonProperty("url") + private String url; + + private Stack stack; + private Configuration configuration = new Configuration(); + + @JsonProperty("service_instances") + private Collection serviceInstances = new ArrayList<>(); + + public MpackInstance(String mpackName, String mpackVersion, String url, Stack stack, Configuration configuration) { + this.mpackName = mpackName; + this.mpackVersion = mpackVersion; + this.url = url; + this.stack = stack; + this.configuration = configuration; + } + + public MpackInstance() { } + + public String getMpackName() { + return mpackName; + } + + public void setMpackName(String mpackName) { + this.mpackName = mpackName; + } + + public String getMpackVersion() { + return mpackVersion; + } + + public void setMpackVersion(String mpackVersion) { + this.mpackVersion = mpackVersion; + } + + @JsonIgnore + public String getMpackNameAndVersion() { + return mpackName + "-" + mpackVersion; + } + + @JsonIgnore + public Stack getStack() { + return stack; + } + + public void setStack(Stack stack) { + this.stack = stack; + } + + @JsonIgnore + public Configuration getConfiguration() { + return configuration; + } + + @JsonIgnore + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + public Collection getServiceInstances() { + return serviceInstances; + } + + public void setServiceInstances(Collection serviceInstances) { + this.serviceInstances = serviceInstances; + serviceInstances.forEach(si -> si.setMpackInstance(this)); + } + + public void addServiceInstance(ServiceInstance serviceInstance) { + serviceInstances.add(serviceInstance); + serviceInstance.setMpackInstance(this); + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public BlueprintMpackInstanceEntity toEntity() { + BlueprintMpackInstanceEntity mpackEntity = new BlueprintMpackInstanceEntity(); + mpackEntity.setMpackUri(url); + mpackEntity.setMpackName(mpackName); + mpackEntity.setMpackVersion(mpackVersion); + Collection mpackConfigEntities = + BlueprintImpl.toConfigEntities(configuration, () -> new BlueprintMpackConfigEntity()); + mpackConfigEntities.forEach( configEntity -> configEntity.setMpackInstance(mpackEntity) ); + mpackEntity.setConfigurations(mpackConfigEntities); + + getServiceInstances().forEach(serviceInstance -> { + BlueprintServiceEntity serviceEntity = new BlueprintServiceEntity(); + serviceEntity.setName(serviceInstance.getName()); + serviceEntity.setType(serviceInstance.getType()); + Collection serviceConfigEntities = + BlueprintImpl.toConfigEntities(serviceInstance.getConfiguration(), () -> new BlueprintServiceConfigEntity()); + serviceConfigEntities.forEach( configEntity -> configEntity.setService(serviceEntity) ); + serviceEntity.setConfigurations(serviceConfigEntities); + mpackEntity.getServiceInstances().add(serviceEntity); + serviceEntity.setMpackInstance(mpackEntity); + }); + return mpackEntity; + } + + public static MpackInstance fromEntity(BlueprintMpackInstanceEntity entity) { + MpackInstance mpack = new MpackInstance(); + mpack.setUrl(entity.getMpackUri()); + mpack.setMpackName(entity.getMpackName()); + mpack.setMpackVersion(entity.getMpackVersion()); + mpack.setConfiguration(BlueprintImpl.fromConfigEntities(entity.getConfigurations())); + for (BlueprintServiceEntity serviceEntity: entity.getServiceInstances()) { + ServiceInstance serviceInstance = new ServiceInstance(); + serviceInstance.setName(serviceEntity.getName()); + serviceInstance.setType(serviceEntity.getType()); + serviceInstance.setMpackInstance(mpack); + serviceInstance.setConfiguration(BlueprintImpl.fromConfigEntities(serviceEntity.getConfigurations())); + mpack.getServiceInstances().add(serviceInstance); + } + return mpack; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java new file mode 100644 index 00000000000..8fcee980d21 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceInstance.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public class ServiceInstance implements Configurable { + private String name; + private String type; + private Configuration configuration = new Configuration(); + private MpackInstance mpackInstance; + + public ServiceInstance() { } + + public ServiceInstance(String name, String type, Configuration configuration) { + this.name = name; + this.type = type; + this.configuration = configuration; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @JsonIgnore + public Configuration getConfiguration() { + return configuration; + } + + @JsonIgnore + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + @JsonIgnore + public MpackInstance getMpackInstance() { + return mpackInstance; + } + + @JsonIgnore + void setMpackInstance(MpackInstance mpackInstance) { + this.mpackInstance = mpackInstance; + } +} diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java index 6da76713b5c..35add3c203c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/TopologyManager.java @@ -275,7 +275,7 @@ public RequestStatusResponse provisionCluster(final ProvisionClusterRequest requ final ClusterTopology topology = new ClusterTopologyImpl(ambariContext, request); final String clusterName = request.getClusterName(); - final Stack stack = topology.getBlueprint().getStack(); + final Stack stack = topology.getBlueprint().getStack(); // TODO: implement multi-stack final String repoVersion = request.getRepositoryVersion(); final Long repoVersionID = request.getRepositoryVersionId(); @@ -1095,7 +1095,7 @@ private boolean isHostIgnored(String host) { */ private void addKerberosClient(ClusterTopology topology) { for (HostGroup group : topology.getBlueprint().getHostGroups().values()) { - group.addComponent("KERBEROS_CLIENT"); + group.addComponent(new Component("KERBEROS_CLIENT")); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java index 990aee77687..74fb9b1c897 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/tasks/PersistHostResourcesTask.java @@ -52,7 +52,7 @@ public void runTask() { HostGroup group = hostRequest.getHostGroup(); Map> serviceComponents = new HashMap<>(); for (String service : group.getServices()) { - serviceComponents.put(service, new HashSet<>(group.getComponents(service))); + serviceComponents.put(service, new HashSet<>(group.getComponentNames(service))); } clusterTopology.getAmbariContext().createAmbariHostResources(hostRequest.getClusterId(), hostRequest.getHostName(), serviceComponents); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java index 80b259369c0..9d9ad00a35b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/HiveServiceValidator.java @@ -59,7 +59,7 @@ public void validate(ClusterTopology topology) throws InvalidTopologyException { } // hive database settings need the mysql-server component in the blueprint - if (!topology.getBlueprint().getComponents(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) { + if (!topology.getBlueprint().getComponentNames(HIVE_SERVICE).contains(MYSQL_SERVER_COMPONENT)) { String errorMessage = String.format("Component [%s] must explicitly be set in the blueprint when hive database " + "is configured with the current settings. HIVE service validation failed.", MYSQL_SERVER_COMPONENT); LOGGER.error(errorMessage); diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index 3d04e48df7f..c45da45e3ae 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -618,7 +618,6 @@ CREATE TABLE blueprint_mpack_instance( CONSTRAINT FK_mpi_blueprint_name FOREIGN KEY (blueprint_name) REFERENCES blueprint(blueprint_name), CONSTRAINT FK_mpi_mpack_id FOREIGN KEY (mpack_id) REFERENCES mpacks(id)); - CREATE TABLE blueprint_service ( id BIGINT NOT NULL, mpack_instance_id BIGINT NOT NULL, diff --git a/ambari-server/src/main/resources/properties.json b/ambari-server/src/main/resources/properties.json index a9950490877..606af93d99b 100644 --- a/ambari-server/src/main/resources/properties.json +++ b/ambari-server/src/main/resources/properties.json @@ -377,7 +377,8 @@ "host_groups/cardinality", "configurations", "validate_topology", - "settings" + "settings", + "mpack_instances" ], "Recommendation":[ "Recommendation/id", diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index 3f6de7d3340..d9d674d096b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -156,6 +156,35 @@ public void init() throws Exception { Collections.emptyMap()).anyTimes(); expect(serviceInfo.getRequiredServices()).andReturn(Collections.emptyList()).anyTimes(); + setupGetServiceForComponentExpectations(); + + expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); + + Set emptySet = Collections.emptySet(); + expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); + + expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); + expect(configHelper.getDefaultStackProperties( + EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes(); + + stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties); + + + expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once(); + expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes(); + PowerMock.mockStatic(AmbariServer.class); + expect(AmbariServer.getController()).andReturn(controller).anyTimes(); + PowerMock.replay(AmbariServer.class); + expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes(); + expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes(); + expect(controller.getClusters()).andReturn(clusters).anyTimes(); + expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes(); + Set properties = new HashSet<>(); + properties.add("core-site/hadoop.security.auth_to_local"); + expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes(); + } + + private void setupGetServiceForComponentExpectations() { Collection hdfsComponents = new HashSet<>(); hdfsComponents.add("NAMENODE"); hdfsComponents.add("SECONDARY_NAMENODE"); @@ -163,12 +192,8 @@ public void init() throws Exception { hdfsComponents.add("HDFS_CLIENT"); serviceComponents.put("HDFS", hdfsComponents); - Collection yarnComponents = new HashSet<>(); - yarnComponents.add("RESOURCEMANAGER"); - yarnComponents.add("NODEMANAGER"); - yarnComponents.add("YARN_CLIENT"); - yarnComponents.add("APP_TIMELINE_SERVER"); - serviceComponents.put("YARN", yarnComponents); + serviceComponents.put("YARN", Sets.newHashSet("RESOURCEMANAGER", "NODEMANAGER", "YARN_CLIENT", + "APP_TIMELINE_SERVER", "HISTORYSERVER")); Collection mrComponents = new HashSet<>(); mrComponents.add("MAPREDUCE2_CLIENT"); @@ -180,11 +205,7 @@ public void init() throws Exception { zkComponents.add("ZOOKEEPER_CLIENT"); serviceComponents.put("ZOOKEEPER", zkComponents); - Collection hiveComponents = new HashSet<>(); - hiveComponents.add("MYSQL_SERVER"); - hiveComponents.add("HIVE_METASTORE"); - hiveComponents.add("HIVE_SERVER"); - serviceComponents.put("HIVE", hiveComponents); + serviceComponents.put("HIVE", Sets.newHashSet("MYSQL_SERVER", "HIVE_METASTORE", "HIVE_SERVER", "HIVE_CLIENT")); Collection falconComponents = new HashSet<>(); falconComponents.add("FALCON_SERVER"); @@ -222,9 +243,11 @@ public void init() throws Exception { amsComponents.add("METRICS_COLLECTOR"); serviceComponents.put("AMBARI_METRICS", amsComponents); - Collection stormComponents = new HashSet<>(); - stormComponents.add("NIMBUS"); - serviceComponents.put("STORM", stormComponents); + serviceComponents.put("STORM", Sets.newHashSet("NIMBUS", "SUPERVISOR", "STORM_UI_SERVER", "DRPC_SERVER")); + serviceComponents.put("RANGER", Sets.newHashSet("RANGER_ADMIN", "RANGER_USERSYNC", "RANGER_KMS_SERVER")); + serviceComponents.put("DRUID", Sets.newHashSet("DRUID_COORDINATOR", "DRUID_OVERLORD", "DRUID_BROKER", "DRUID_ROUTER")); + serviceComponents.put("HAWQ", Sets.newHashSet("HAWQMASTER", "HAWQSTANDBY")); + serviceComponents.put("TEZ", Sets.newHashSet("TEZ_CLIENT")); for (Map.Entry> entry : serviceComponents.entrySet()) { String service = entry.getKey(); @@ -232,31 +255,6 @@ public void init() throws Exception { expect(stack.getServiceForComponent(component)).andReturn(service).anyTimes(); } } - - expect(stack.getCardinality("MYSQL_SERVER")).andReturn(new Cardinality("0-1")).anyTimes(); - - Set emptySet = Collections.emptySet(); - expect(stack.getExcludedConfigurationTypes(anyObject(String.class))).andReturn(emptySet).anyTimes(); - - expect(ambariContext.getConfigHelper()).andReturn(configHelper).anyTimes(); - expect(configHelper.getDefaultStackProperties( - EasyMock.eq(new StackId(STACK_NAME, STACK_VERSION)))).andReturn(stackProperties).anyTimes(); - - stackProperties.put(ConfigHelper.CLUSTER_ENV, defaultClusterEnvProperties); - - - expect(ambariContext.isClusterKerberosEnabled(1)).andReturn(true).once(); - expect(ambariContext.getClusterName(1L)).andReturn("clusterName").anyTimes(); - PowerMock.mockStatic(AmbariServer.class); - expect(AmbariServer.getController()).andReturn(controller).anyTimes(); - PowerMock.replay(AmbariServer.class); - expect(clusters.getCluster("clusterName")).andReturn(cluster).anyTimes(); - expect(controller.getKerberosHelper()).andReturn(kerberosHelper).anyTimes(); - expect(controller.getClusters()).andReturn(clusters).anyTimes(); - expect(kerberosHelper.getKerberosDescriptor(cluster, false)).andReturn(kerberosDescriptor).anyTimes(); - Set properties = new HashSet<>(); - properties.add("core-site/hadoop.security.auth_to_local"); - expect(kerberosDescriptor.getAllAuthToLocalProperties()).andReturn(properties).anyTimes(); } @After @@ -4275,6 +4273,10 @@ public void testExcludedPropertiesShouldBeAddedWhenServiceIsInBlueprint() throws expect(stack.getExcludedConfigurationTypes("OOZIE")).andReturn(Collections.emptySet()); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4314,6 +4316,10 @@ public void testExcludedPropertiesShouldBeIgnoredWhenServiceIsNotInBlueprint() t expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4350,6 +4356,8 @@ public void testAddExcludedPropertiesAreOverwrittenByBlueprintConfigs() throws E // customized stack calls for this test only expect(stack.getExcludedConfigurationTypes("FALCON")).andReturn(Collections.singleton("oozie-site")).anyTimes(); expect(stack.getConfigurationProperties("FALCON", "oozie-site")).andReturn(Collections.singletonMap("oozie.service.ELService.ext.functions.coord-job-submit-instances", "testValue")).anyTimes(); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); Map> properties = new HashMap<>(); Map typeProps = new HashMap<>(); @@ -4397,6 +4405,10 @@ public void testExcludedPropertiesHandlingWhenExcludedConfigServiceIsNotFoundInS expect(stack.getServiceForConfigType("oozie-site")).andReturn("OOZIE").anyTimes(); // simulate the case where the STORM service has been removed manually from the stack definitions expect(stack.getServiceForConfigType("storm-site")).andThrow(new IllegalArgumentException("TEST: Configuration not found in stack definitions!")); + expect(stack.getServiceForComponent("FALCON_CLIENT")).andReturn("FALCON"); + expect(stack.getServiceForComponent("FALCON_SERVER")).andReturn("FALCON"); + expect(stack.getServiceForComponent("OOZIE_CLIENT")).andReturn("OOZIE"); + expect(stack.getServiceForComponent("OOZIE_SERVER")).andReturn("OOZIE"); Map> properties = new HashMap<>(); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4593,6 +4605,8 @@ Set getDependsOnProperties() { expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS").anyTimes(); + Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); Collection hgComponents = new HashSet<>(); @@ -4670,6 +4684,7 @@ Set getDependsOnProperties() { // simulate the case of the stack object throwing a RuntimeException, to indicate a config error expect(stack.getServiceForConfigType("hive-site")).andThrow(new RuntimeException("Expected Test Error")).once(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS"); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4748,6 +4763,7 @@ Set getDependsOnProperties() { // customized stack calls for this test only expect(stack.getServiceForConfigType("hive-site")).andReturn("HIVE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HIVE", "hive-site")).andReturn(mapOfMetadata).atLeastOnce(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS"); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4836,6 +4852,7 @@ Set getDependsOnProperties() { // customized stack calls for this test only expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS"); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -4904,6 +4921,7 @@ Set getDependsOnProperties() { // customized stack calls for this test only expect(stack.getServiceForConfigType("hbase-site")).andReturn("HBASE").atLeastOnce(); expect(stack.getConfigurationPropertiesWithMetadata("HBASE", "hbase-site")).andReturn(mapOfMetadata).atLeastOnce(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS"); Configuration clusterConfig = new Configuration(properties, Collections.emptyMap()); @@ -8214,7 +8232,7 @@ private ClusterTopology createClusterTopology(Blueprint blueprint, Configuration } //create host group which is set on topology - allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", stack, + allHostGroups.put(hostGroup.name, new HostGroupImpl(hostGroup.name, "test-bp", Collections.singleton(stack), componentList, EMPTY_CONFIG, "1")); hostGroupInfo.put(hostGroup.name, groupInfo); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java index fc61a204a41..e303731d35d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintResourceProviderTest.java @@ -109,7 +109,7 @@ public class BlueprintResourceProviderTest { @BeforeClass public static void initClass() { - BlueprintResourceProvider.init(blueprintFactory, dao, securityFactory, gson, metaInfo); + BlueprintResourceProvider.init(blueprintFactory, dao, securityFactory, metaInfo); } private Map>> getSettingProperties() { diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java index e1f5583c4d8..e70901aaf22 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ExportBlueprintRequestTest.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.controller.internal; import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; @@ -28,8 +29,11 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.powermock.api.easymock.PowerMock.mockStatic; import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -49,11 +53,17 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; /** * ExportBlueprintRequest unit tests. */ @SuppressWarnings("unchecked") +@RunWith(PowerMockRunner.class) +@PrepareForTest({ExportBlueprintRequest.ExportedHostGroup.class}) public class ExportBlueprintRequestTest { private static final String CLUSTER_NAME = "c1"; private static final String CLUSTER_ID = "2"; @@ -70,8 +80,12 @@ public void setupTest() throws Exception { Collections.emptySet()).anyTimes(); expect(controller.getStackLevelConfigurations((Set) anyObject())).andReturn( Collections.emptySet()).anyTimes(); - replay(controller); + + // This can save precious time + mockStatic(InetAddress.class); + expect(InetAddress.getByName(anyString())).andThrow(new UnknownHostException()).anyTimes(); + PowerMock.replay(InetAddress.class); } @After diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java index 5bbf75eff30..0e4f61a284b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/entities/BlueprintEntityTest2.java @@ -34,7 +34,6 @@ import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.dao.BlueprintDAO; -import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.state.SecurityType; import org.junit.After; import org.junit.Before; @@ -54,8 +53,6 @@ public class BlueprintEntityTest2 { @Inject private Injector injector; @Inject - private StackDAO stackDAO; - @Inject private BlueprintDAO blueprintDAO; diff --git a/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java b/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java index f83c46a5e50..312c905d273 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/security/encryption/CredentialStoreTest.java @@ -30,7 +30,7 @@ import junit.framework.Assert; -public class CredentialStoreTest { +public class CredentialStoreTest { @Rule public TemporaryFolder tmpFolder = new TemporaryFolder(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java index 6ee1c5d528e..1bd79e190fa 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/AmbariContextTest.java @@ -243,8 +243,8 @@ public void setUp() throws Exception { expect(blueprint.getName()).andReturn(BP_NAME).anyTimes(); expect(blueprint.getStack()).andReturn(stack).anyTimes(); expect(blueprint.getServices()).andReturn(blueprintServices).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); + expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("s1Component1", "s1Component2")).anyTimes(); + expect(blueprint.getComponentNames("service2")).andReturn(Collections.singleton("s2Component1")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getCredentialStoreEnabled("service1")).andReturn("true").anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java index e7c0fe91579..c2bf712ae93 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintFactoryTest.java @@ -18,13 +18,18 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.anyString; import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.powermock.api.easymock.PowerMock.createStrictMock; +import static org.powermock.api.easymock.PowerMock.expectNew; import static org.powermock.api.easymock.PowerMock.replay; import static org.powermock.api.easymock.PowerMock.reset; import static org.powermock.api.easymock.PowerMock.verify; @@ -35,8 +40,10 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.apache.ambari.server.ObjectNotFoundException; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.BlueprintResourceProvider; import org.apache.ambari.server.controller.internal.BlueprintResourceProviderTest; import org.apache.ambari.server.controller.internal.Stack; @@ -48,11 +55,20 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; /** * BlueprintFactory unit tests. */ @SuppressWarnings("unchecked") +@RunWith(PowerMockRunner.class) +@PrepareForTest(BlueprintImpl.class) public class BlueprintFactoryTest { private static final String BLUEPRINT_NAME = "test-blueprint"; @@ -107,6 +123,42 @@ public void tearDown() { // // } + @Test + public void testGetMultiInstanceBlueprint() throws Exception { + // prepare + Blueprint expectedBlueprint = createMultiInstanceBlueprint(); + + reset(dao); + expect(dao.findByName(BLUEPRINT_NAME)).andReturn(expectedBlueprint.toEntity()); + Stack hdpStack = createNiceMock(Stack.class); + expect(hdpStack.getName()).andReturn("HDPCORE").anyTimes(); + expect(hdpStack.getVersion()).andReturn("3.0.0.0").anyTimes(); + Stack edwStack = createNiceMock(Stack.class); + expect(edwStack.getName()).andReturn("EDW").anyTimes(); + expect(edwStack.getVersion()).andReturn("3.1.0.0").anyTimes(); + expectNew(Stack.class, eq("HDPCORE-3.0"), anyString(), anyObject(AmbariManagementController.class)).andReturn(hdpStack).anyTimes(); + expectNew(Stack.class, eq("EDW-3.1"), anyString(), anyObject(AmbariManagementController.class)).andReturn(edwStack).anyTimes(); + replay(Stack.class, hdpStack, edwStack, dao); + + // test + Blueprint blueprint = testFactory.getBlueprint(BLUEPRINT_NAME); + Set mpackNames = + blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(Collectors.toSet()); + assertEquals(ImmutableSet.of("HDPCORE-3.0", "EDW-3.1"), mpackNames ); + MpackInstance hdpCore = + blueprint.getMpacks().stream().filter(mp -> "HDPCORE-3.0".equals(mp.getMpackName())).findAny().get(); + Set serviceInstanceNames = + hdpCore.getServiceInstances().stream().map(ServiceInstance::getName).collect(toSet()); + assertEquals(ImmutableSet.of("ZK1", "ZK2"), serviceInstanceNames); + Set serviceInstanceTypes = + hdpCore.getServiceInstances().stream().map(ServiceInstance::getType).collect(toSet()); + assertEquals(ImmutableSet.of("ZOOKEEPER"), serviceInstanceTypes); + Set stackNames = + blueprint.getStacks().stream().map(Stack::getName).collect(Collectors.toSet()); + assertEquals(ImmutableSet.of("HDPCORE", "EDW"), stackNames); + assertEquals(1, blueprint.getHostGroups().size()); + } + @Test public void testGetBlueprint_NotFound() throws Exception { expect(dao.findByName(BLUEPRINT_NAME)).andReturn(null).once(); @@ -163,6 +215,43 @@ public void testCreateBlueprint() throws Exception { verify(dao, entity, configEntity); } + @Test + public void testCreateMultiInstanceBlueprint() throws Exception { + createMultiInstanceBlueprint(); + } + + public Blueprint createMultiInstanceBlueprint() throws Exception { + Map> allComponents = ImmutableMap.of( + "HDFS", ImmutableSet.of("NAMENODE", "SECONDARY_NAMENODE"), + "ZOOKEEPER", ImmutableSet.of("ZOOKEEPER_SERVER") + ); + reset(stack); + expect(stack.getComponents()).andReturn(allComponents).anyTimes(); + expect(stack.isMasterComponent("NAMENODE")).andReturn(true).anyTimes(); + expect(stack.isMasterComponent("ZOOKEEPER_SERVER")).andReturn(true).anyTimes(); + expect(stack.isMasterComponent("SECONDAY_NAMENODE")).andReturn(false).anyTimes(); + expect(stack.getServiceForComponent("NAMENODE")).andReturn("HDFS").anyTimes(); + expect(stack.getServiceForComponent("SECONDARY_NAMENODE")).andReturn("HDFS").anyTimes(); + expect(stack.getServiceForComponent("ZOOKEEPER_SERVER")).andReturn("ZOOKEEPER").anyTimes(); + + replay(stack, dao, entity, configEntity); + + Map props = BlueprintTestUtil.getMultiInstanceBlueprintAsMap(); + Blueprint blueprint = testFactory.createBlueprint(props, null); + + assertEquals(2, blueprint.getMpacks().size()); + assertEquals(Sets.newHashSet("HDPCORE-3.0", "EDW-3.1"), + blueprint.getMpacks().stream().map(MpackInstance::getMpackName).collect(toSet())); + MpackInstance hdpCore = + blueprint.getMpacks().stream().filter( mpack -> "HDPCORE-3.0".equals(mpack.getMpackName()) ).findFirst().get(); + assertEquals(2, hdpCore.getServiceInstances().size()); + ServiceInstance zk1 = + hdpCore.getServiceInstances().stream().filter( si -> "ZK1".equals(si.getName()) ).findFirst().get(); + assertEquals("ZOOKEEPER", zk1.getType()); + assertEquals("/zookeeper1", zk1.getConfiguration().getProperties().get("zoo.cfg").get("dataDir")); + return blueprint; + } + @Test(expected=NoSuchStackException.class) public void testCreateInvalidStack() throws Exception { EasyMockSupport mockSupport = new EasyMockSupport(); @@ -240,7 +329,7 @@ public TestBlueprintFactory(Stack stack) { } @Override - protected Stack createStack(Map properties) throws NoSuchStackException { + protected Stack loadStack(String stackName, String stackVersion) throws NoSuchStackException { return stack; } } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java index d34526a8b3e..057332d4013 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java @@ -124,7 +124,7 @@ public void testValidateConfigurations__basic_positive() throws Exception { category2Props.put("prop2", "val"); SecurityConfiguration securityConfiguration = new SecurityConfiguration(SecurityType.KERBEROS, "testRef", null); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, securityConfiguration); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, securityConfiguration, null); blueprint.validateRequiredProperties(); BlueprintEntity entity = blueprint.toEntity(); @@ -162,7 +162,7 @@ public void testValidateConfigurations__hostGroupConfig() throws Exception { hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP:group1%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP:group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null); blueprint.validateRequiredProperties(); BlueprintEntity entity = blueprint.toEntity(); verify(stack, group1, group2, serverConfig); @@ -202,7 +202,7 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAPositive() t hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null); blueprint.validateRequiredProperties(); BlueprintEntity entity = blueprint.toEntity(); @@ -243,7 +243,7 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHos hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group3%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @@ -279,7 +279,7 @@ public void testValidateConfigurations__hostGroupConfigForNameNodeHAMappedSameHo hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group2%"); hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @@ -294,7 +294,7 @@ public void testValidateConfigurations__secretReference() throws InvalidTopology hdfsProps.put("secret", "SECRET:hdfs-site:1:test"); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @@ -311,7 +311,7 @@ public void testValidateConfigurations__gplIsNotAllowedCodecsProperty() throws I serverConfig = setupConfigurationWithGPLLicense(false); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @@ -328,7 +328,7 @@ public void testValidateConfigurations__gplIsNotAllowedLZOProperty() throws Inva serverConfig = setupConfigurationWithGPLLicense(false); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @@ -346,32 +346,36 @@ public void testValidateConfigurations__gplISAllowed() throws InvalidTopologyExc expect(group2.getConfiguration()).andReturn(EMPTY_CONFIGURATION).atLeastOnce(); replay(stack, group1, group2, serverConfig); - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, lzoUsageConfiguration, null, null); blueprint.validateRequiredProperties(); verify(stack, group1, group2, serverConfig); } @Test public void testAutoSkipFailureEnabled() { - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting); HashMap skipFailureSetting = new HashMap<>(); skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "true"); + expect(stack.getName()).andReturn("HDPCORE").anyTimes(); + expect(stack.getVersion()).andReturn("3.0.0.0").anyTimes(); expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); replay(stack, setting); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting); assertTrue(blueprint.shouldSkipFailure()); + verify(stack, setting); } @Test public void testAutoSkipFailureDisabled() { - Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting); HashMap skipFailureSetting = new HashMap<>(); skipFailureSetting.put(Setting.SETTING_NAME_SKIP_FAILURE, "false"); expect(setting.getSettingValue(Setting.SETTING_NAME_DEPLOYMENT_SETTINGS)).andReturn(Collections.singleton(skipFailureSetting)); replay(stack, setting); + Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null, setting); assertFalse(blueprint.shouldSkipFailure()); + verify(stack, setting); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java new file mode 100644 index 00000000000..1bc372398a9 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintTestUtil.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import java.io.IOException; +import java.net.URL; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.io.Resources; + +public class BlueprintTestUtil { + public static final String BLUEPRINT_30_LOCATION = "blueprint3.0/multi-instance-blueprint.json"; + + public static Map getMultiInstanceBlueprintAsMap() { + URL url = Resources.getResource(BLUEPRINT_30_LOCATION); + try { + Map blueprintMap = new ObjectMapper().readValue(url, new TypeReference>(){}); + + Map blueprintProps = (Map) blueprintMap.remove("Blueprints"); + blueprintProps.forEach( (key, value) -> blueprintMap.put("Blueprints/" + key, value)); + Map settingProps = (Map) blueprintMap.remove("settings"); + settingProps.forEach( (key, value) -> blueprintMap.put("settings/" + key, value)); + + return convertListsToSets(blueprintMap); + } + catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + private static Map convertListsToSets(Map input) { + Map converted = new HashMap<>(); + input.forEach( (k, v) -> { + if (v instanceof List) { + List list = (List)v; + Set set = list.stream().map(e -> { + if (e instanceof Map) { + return convertListsToSets((Map)e); + } + else { + return e; + } + }).collect(Collectors.toSet()); + converted.put(k, set); + } + else if (v instanceof Map){ + converted.put(k, convertListsToSets((Map)v)); + } + else { + converted.put(k, v); + } + }); + return converted; + } + + +} diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java index 75a9d6b238a..e88bef3d237 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toList; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.reset; @@ -103,8 +104,12 @@ public void setup() { expect(blueprint.getServices()).andReturn(services).anyTimes(); expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); + expect(group1.getComponents()). + andAnswer(() -> group1Components.stream().map(comp -> new Component(comp)).collect(toList())).anyTimes(); expect(group1.getName()).andReturn("host-group-1").anyTimes(); expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); + expect(group2.getComponents()). + andAnswer(() -> group2Components.stream().map(comp -> new Component(comp)).collect(toList())).anyTimes(); expect(group2.getName()).andReturn("host-group-2").anyTimes(); expect(stack.getDependenciesForComponent("component1")).andReturn(dependencies1).anyTimes(); @@ -119,6 +124,7 @@ public void setup() { dependenciesConditionInfos1.add(dependencyConditionInfo2); expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes(); + expect(blueprint.isAllMpacksResolved()).andReturn(true); } @After @@ -171,7 +177,7 @@ public void testValidateTopology_autoDeploy() throws Exception { expect(stack.getComponents("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes(); expect(stack.getAutoDeployInfo("component1")).andReturn(autoDeploy).anyTimes(); - expect(group1.addComponent("component1")).andReturn(true).once(); + expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); replay(blueprint, stack, group1, group2, dependency1); BlueprintValidator validator = new BlueprintValidatorImpl(blueprint); @@ -207,8 +213,8 @@ public void testValidateTopology_autoDeploy_hasDependency() throws Exception { expect(dependencyComponentInfo.isClient()).andReturn(true).anyTimes(); expect(stack.getComponentInfo("component3")).andReturn(dependencyComponentInfo).anyTimes(); - expect(group1.addComponent("component1")).andReturn(true).once(); - expect(group1.addComponent("component3")).andReturn(true).once(); + expect(group1.addComponent(new Component("component1"))).andReturn(true).once(); + expect(group1.addComponent(new Component("component3"))).andReturn(true).once(); replay(blueprint, stack, group1, group2, dependency1, dependencyComponentInfo); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java index 771b89fa238..28ce72a167f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java @@ -257,9 +257,9 @@ private Capture> testProcessWithKerberos(String blueprintP List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); @@ -346,9 +346,9 @@ public void testProcessClusterConfigRequestDontIncludeKererosConfigs() throws Ex List zookeeperComponents = new ArrayList<>(); zookeeperComponents.add("ZOOKEEPER_SERVER"); - expect(blueprint.getComponents("HDFS")).andReturn(hdfsComponents).anyTimes(); - expect(blueprint.getComponents("KERBEROS")).andReturn(kerberosComponents).anyTimes(); - expect(blueprint.getComponents("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); + expect(blueprint.getComponentNames("HDFS")).andReturn(hdfsComponents).anyTimes(); + expect(blueprint.getComponentNames("KERBEROS")).andReturn(kerberosComponents).anyTimes(); + expect(blueprint.getComponentNames("ZOOKEPER")).andReturn(zookeeperComponents).anyTimes(); expect(topology.getAmbariContext()).andReturn(ambariContext).anyTimes(); expect(topology.getConfigRecommendationStrategy()).andReturn(ConfigRecommendationStrategy.NEVER_APPLY).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java index aecc6cb6403..d0e82f21a56 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterDeployWithStartOnlyTest.java @@ -241,8 +241,8 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -304,8 +304,8 @@ public void setup() throws Exception { expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); @@ -316,8 +316,8 @@ public void setup() throws Exception { expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); + expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); + expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java index a4b2160a539..c64e2281100 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartOnComponentLevelTest.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toList; import static org.apache.ambari.server.controller.internal.ProvisionAction.INSTALL_AND_START; import static org.easymock.EasyMock.anyBoolean; import static org.easymock.EasyMock.anyLong; @@ -182,10 +183,11 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private Collection group1Components = Arrays.asList("component1", "component2", "component3"); private Collection group2Components = Arrays.asList("component3", "component4"); - private Map> group1ServiceComponents = new HashMap<>(); - private Map> group2ServiceComponents = new HashMap<>(); + private Map> group1ServiceComponents = new HashMap<>(); + private Map> group2ServiceComponents = new HashMap<>(); - private Map> serviceComponents = new HashMap<>(); + private Map> serviceComponents = new HashMap<>(); + private Map> serviceComponentNames = new HashMap<>(); private String predicate = "Hosts/host_name=foo"; @@ -198,6 +200,7 @@ public class ClusterInstallWithoutStartOnComponentLevelTest extends EasyMockSupp private Capture updateClusterConfigRequestCapture; private Capture updateConfigTaskCapture; + @Before public void setup() throws Exception { clusterTopologyCapture = newCapture(); @@ -227,18 +230,22 @@ public void setup() throws Exception { groupMap.put("group1", group1); groupMap.put("group2", group2); - serviceComponents.put("service1", Arrays.asList("component1", "component3")); - serviceComponents.put("service2", Arrays.asList("component2", "component4")); + serviceComponents.put("service1", Arrays.asList(new Component("component1"), new Component("component3"))); + serviceComponents.put("service2", Arrays.asList(new Component("component2"), new Component("component4"))); + + for(Map.Entry> entry: serviceComponents.entrySet()) { + serviceComponentNames.put(entry.getKey(), entry.getValue().stream().map(Component::getName).collect(toList())); + } - group1ServiceComponents.put("service1", Arrays.asList("component1", "component3")); - group1ServiceComponents.put("service2", Collections.singleton("component2")); - group2ServiceComponents.put("service2", Collections.singleton("component3")); - group2ServiceComponents.put("service2", Collections.singleton("component4")); + group1ServiceComponents.put("service1", Arrays.asList(new Component("component1"), new Component("component3"))); + group1ServiceComponents.put("service2", Collections.singleton(new Component("component2"))); + group2ServiceComponents.put("service2", Collections.singleton(new Component("component3"))); + group2ServiceComponents.put("service2", Collections.singleton(new Component("component4"))); expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getComponents("service1")).andReturn(Arrays.asList(new Component("component1"), new Component("component3"))).anyTimes(); + expect(blueprint.getComponents("service2")).andReturn(Arrays.asList(new Component("component2"), new Component("component4"))).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -273,9 +280,9 @@ public void setup() throws Exception { expect(stack.getCardinality("component2")).andReturn(new Cardinality("1")).anyTimes(); expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+")).anyTimes(); expect(stack.getCardinality("component4")).andReturn(new Cardinality("1+")).anyTimes(); - expect(stack.getComponents()).andReturn(serviceComponents).anyTimes(); - expect(stack.getComponents("service1")).andReturn(serviceComponents.get("service1")).anyTimes(); - expect(stack.getComponents("service2")).andReturn(serviceComponents.get("service2")).anyTimes(); + expect(stack.getComponents()).andReturn(serviceComponentNames).anyTimes(); + expect(stack.getComponents("service1")).andReturn(serviceComponentNames.get("service1")).anyTimes(); + expect(stack.getComponents("service2")).andReturn(serviceComponentNames.get("service2")).anyTimes(); expect(stack.getConfiguration()).andReturn(stackConfig).anyTimes(); expect(stack.getName()).andReturn(STACK_NAME).anyTimes(); expect(stack.getVersion()).andReturn(STACK_VERSION).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java index d89c8ca2542..6fa70b437a2 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java @@ -239,8 +239,8 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -303,8 +303,8 @@ public void setup() throws Exception { expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group1.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); @@ -315,8 +315,8 @@ public void setup() throws Exception { expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); expect(group2.getComponentNames(anyObject(ProvisionAction.class))).andReturn(Collections.emptyList()).anyTimes(); - expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); + expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); + expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java new file mode 100644 index 00000000000..dfcee8475ca --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ConfigurableTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.topology; + +import static org.junit.Assert.assertEquals; + +import java.net.URL; + +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Resources; + +public class ConfigurableTest { + public static final String JSON_LOCATION = "blueprint3.0/configurable.json"; + public static final String JSON_LOCATION2 = "blueprint3.0/configurable2.json"; + + private TestConfigurable configurable; + private ObjectMapper mapper; + + @Before + public void setUp() throws Exception { + mapper = new ObjectMapper(); + URL url = Resources.getResource(JSON_LOCATION); + configurable = new ObjectMapper().readValue(url, TestConfigurable.class); + } + + @Test + public void testParseConfigurable() throws Exception{ + assertEquals(ImmutableMap.of("zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")), + configurable.getConfiguration().getProperties()); + assertEquals( + ImmutableMap.of("zoo.cfg", + ImmutableMap.of("final", + ImmutableMap.of("someProp", "someValue"))), + configurable.getConfiguration().getAttributes()); + } + + @Test + public void testSerializaDeserialize() throws Exception { + String persisted = mapper.writeValueAsString(configurable); + Configurable restored = mapper.readValue(persisted, TestConfigurable.class); + assertEquals(configurable.getConfiguration().getProperties(), restored.getConfiguration().getProperties()); + assertEquals(configurable.getConfiguration().getAttributes(), restored.getConfiguration().getAttributes()); + } + + @Test + public void testParseConfigurableFromResoueceManager() throws Exception{ + mapper = new ObjectMapper(); + URL url = Resources.getResource(JSON_LOCATION2); + configurable = new ObjectMapper().readValue(url, TestConfigurable.class); + + assertEquals(ImmutableMap.of("zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")), + configurable.getConfiguration().getProperties()); + assertEquals( + ImmutableMap.of("zoo.cfg", + ImmutableMap.of("final", + ImmutableMap.of("someProp", "someValue"))), + configurable.getConfiguration().getAttributes()); + } + + +} + +class TestConfigurable implements Configurable { + Configuration configuration; + + @Override + public Configuration getConfiguration() { + return configuration; + } + + @Override + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + + +} \ No newline at end of file diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java index c474493eca4..722f5f1e31c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/RequiredPasswordValidatorTest.java @@ -149,12 +149,12 @@ public void setup() { expect(group1.getComponentNames()).andReturn(group1Components).anyTimes(); expect(group2.getComponentNames()).andReturn(group2Components).anyTimes(); - expect(group1.getComponents("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(Arrays.asList("component3")).anyTimes(); - expect(group1.getComponents("service3")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponents("service1")).andReturn(Arrays.asList("component1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(Collections.emptySet()).anyTimes(); - expect(group2.getComponents("service3")).andReturn(Arrays.asList("component4")).anyTimes(); + expect(group1.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component2")).anyTimes(); + expect(group1.getComponentNames("service2")).andReturn(Arrays.asList("component3")).anyTimes(); + expect(group1.getComponentNames("service3")).andReturn(Collections.emptySet()).anyTimes(); + expect(group2.getComponentNames("service1")).andReturn(Arrays.asList("component1")).anyTimes(); + expect(group2.getComponentNames("service2")).andReturn(Collections.emptySet()).anyTimes(); + expect(group2.getComponentNames("service3")).andReturn(Arrays.asList("component4")).anyTimes(); expect(stack.getServiceForComponent("component1")).andReturn("service1").anyTimes(); expect(stack.getServiceForComponent("component2")).andReturn("service1").anyTimes(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java index c12965d3017..0daf20fcf7b 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/TopologyManagerTest.java @@ -244,8 +244,8 @@ public void setup() throws Exception { expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes(); expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes(); - expect(blueprint.getComponents("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); - expect(blueprint.getComponents("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); + expect(blueprint.getComponentNames("service1")).andReturn(Arrays.asList("component1", "component3")).anyTimes(); + expect(blueprint.getComponentNames("service2")).andReturn(Arrays.asList("component2", "component4")).anyTimes(); expect(blueprint.getConfiguration()).andReturn(bpConfiguration).anyTimes(); expect(blueprint.getHostGroups()).andReturn(groupMap).anyTimes(); expect(blueprint.getHostGroupsForComponent("component1")).andReturn(Collections.singleton(group1)).anyTimes(); @@ -295,8 +295,8 @@ public void setup() throws Exception { expect(group1.getCardinality()).andReturn("test cardinality").anyTimes(); expect(group1.containsMasterComponent()).andReturn(true).anyTimes(); expect(group1.getComponents()).andReturn(group1Components).anyTimes(); - expect(group1.getComponents("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); - expect(group1.getComponents("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service1")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); + expect(group1.getComponentNames("service2")).andReturn(group1ServiceComponents.get("service1")).anyTimes(); expect(group1.getConfiguration()).andReturn(topoGroup1Config).anyTimes(); expect(group1.getName()).andReturn("group1").anyTimes(); expect(group1.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); @@ -306,8 +306,8 @@ public void setup() throws Exception { expect(group2.getCardinality()).andReturn("test cardinality").anyTimes(); expect(group2.containsMasterComponent()).andReturn(false).anyTimes(); expect(group2.getComponents()).andReturn(group2Components).anyTimes(); - expect(group2.getComponents("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); - expect(group2.getComponents("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); + expect(group2.getComponentNames("service1")).andReturn(group2ServiceComponents.get("service1")).anyTimes(); + expect(group2.getComponentNames("service2")).andReturn(group2ServiceComponents.get("service2")).anyTimes(); expect(group2.getConfiguration()).andReturn(topoGroup2Config).anyTimes(); expect(group2.getName()).andReturn("group2").anyTimes(); expect(group2.getServices()).andReturn(Arrays.asList("service1", "service2")).anyTimes(); @@ -407,8 +407,8 @@ public void testAddKerberosClientAtTopologyInit() throws Exception { expect(persistedState.getAllRequests()).andReturn(allRequests).anyTimes(); expect(persistedState.getProvisionRequest(CLUSTER_ID)).andReturn(logicalRequest).anyTimes(); expect(ambariContext.isTopologyResolved(CLUSTER_ID)).andReturn(true).anyTimes(); - expect(group1.addComponent("KERBEROS_CLIENT")).andReturn(true).anyTimes(); - expect(group2.addComponent("KERBEROS_CLIENT")).andReturn(true).anyTimes(); + expect(group1.addComponent(new Component("KERBEROS_CLIENT"))).andReturn(true).anyTimes(); + expect(group2.addComponent(new Component("KERBEROS_CLIENT"))).andReturn(true).anyTimes(); replayAll(); diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java index f4c5ca51fcb..3ce0a024813 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/HiveServiceValidatorTest.java @@ -122,7 +122,7 @@ public void testShouldValidationFailWhenDefaultsAreUsedAndMysqlComponentIsMissin Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponents("HIVE")).andReturn(Collections.emptyList()).anyTimes(); + EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(Collections.emptyList()).anyTimes(); EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); @@ -145,7 +145,7 @@ public void testShouldValidationPassWhenDefaultsAreUsedAndMsqlComponentIsListed( Collection configTypes = Arrays.asList("hive-env", "core-site", "hadoop-env"); EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes(); EasyMock.expect(blueprintMock.getServices()).andReturn(blueprintServices).anyTimes(); - EasyMock.expect(blueprintMock.getComponents("HIVE")).andReturn(hiveComponents).anyTimes(); + EasyMock.expect(blueprintMock.getComponentNames("HIVE")).andReturn(hiveComponents).anyTimes(); EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(configurationMock); EasyMock.expect(configurationMock.getAllConfigTypes()).andReturn(configTypes); diff --git a/ambari-server/src/test/resources/blueprint3.0/configurable.json b/ambari-server/src/test/resources/blueprint3.0/configurable.json new file mode 100644 index 00000000000..9bac290e3ee --- /dev/null +++ b/ambari-server/src/test/resources/blueprint3.0/configurable.json @@ -0,0 +1,16 @@ +{ + "configurations" : [ + { + "zoo.cfg" : { + "properties" : { + "dataDir" : "/zookeeper1" + }, + "properties_attributes": { + "final": { + "someProp": "someValue" + } + } + } + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/test/resources/blueprint3.0/configurable2.json b/ambari-server/src/test/resources/blueprint3.0/configurable2.json new file mode 100644 index 00000000000..fb813ed8d3a --- /dev/null +++ b/ambari-server/src/test/resources/blueprint3.0/configurable2.json @@ -0,0 +1,8 @@ +{ + "configurations" : [ + { + "zoo.cfg/properties/dataDir" : "/zookeeper1", + "zoo.cfg/properties_attributes/final/someProp": "someValue" + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json b/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json new file mode 100644 index 00000000000..d5d21ae03c6 --- /dev/null +++ b/ambari-server/src/test/resources/blueprint3.0/multi-instance-blueprint.json @@ -0,0 +1,104 @@ +{ + "host_groups" : [ + { + "name" : "master", + "components" : [ + { + "name" : "NAMENODE" + }, + { + "name" : "ZOOKEEPER_SERVER", + "mpack-instance": "HDPCORE-3.0", + "service-instance": "ZK1" + }, + { + "name" : "SECONDARY_NAMENODE" + } + ], + "cardinality" : "1" + } + ], + "Blueprints" : { + "blueprint_name" : "test-blueprint", + "stack_name" : "HDP", + "stack_version" : "2.6" + }, + "settings" : { + "deployment_settings": [ + { "skip_failure":"true" } + ], + "repository_settings":[ + { + "override_strategy":"ALWAYS_APPLY", + "operating_system":"redhat7", + "repo_id":"HDP", + "base_url":"http://myserver/hdp" + }, + { + "override_strategy":"APPLY_WHEN_MISSING", + "operating_system":"redhat7", + "repo_id":"HDP-UTIL-1.1", + "base_url":"http://myserver/hdp-util" + } + ], + "recovery_settings":[ + { "recovery_enabled":"true" } + ], + "service_settings":[ + { + "name":"HDFS", + "recovery_enabled":"false" + } + ], + "component_settings":[ + { + "name":"SECONDARY_NAMENODE", + "recovery_enabled" : "true" + } + ] + }, + "configurations": [], + + "mpack_instances" : [ + { + "name" : "HDPCORE-3.0", + "version" : "3.0.0.0", + "url" : "http://www.hortonworks.com/path/to/hdpcore.tar.gz", + "configurations" : [], + "service_instances" : [ + { + "name" : "ZK1", + "type" : "ZOOKEEPER", + "configurations" : [ + { + "zoo.cfg" : { + "properties" : { + "dataDir" : "/zookeeper1" + } + } + } + ], + "settings" : [] + }, + { + "name" : "ZK2", + "type" : "ZOOKEEPER", + "configurations" : [ + { + "zoo.cfg" : { + "properties" : { + "dataDir" : "/zookeeper2" + } + } + } + ] + } + ] + }, + { + "name" : "EDW-3.1", + "version" : "3.1.0.0", + "url" : "http://www.hortonworks.com/path/to/edw.tar.gz" + } + ] +} \ No newline at end of file