diff --git a/scripts/aws/run_systemds_script.sh b/scripts/aws/run_systemds_script.sh index 4c58fcec9bb..c7a461ec7ad 100755 --- a/scripts/aws/run_systemds_script.sh +++ b/scripts/aws/run_systemds_script.sh @@ -32,7 +32,7 @@ fi source systemds_cluster.config -aws s3 cp $1 s3://system-ds-bucket/ --exclude "*" --include "*.dml" +aws s3 cp $1 s3://${BUCKET} --exclude "*" --include "*.dml" if [ ! -z "$2" ] then @@ -41,7 +41,7 @@ fi dml_filename=$(basename $1) -STEP_INFO=$(aws emr add-steps --cluster-id $CLUSTER_ID --steps "Type=Spark, +STEP_INFO=$(aws emr add-steps --cluster-id $CLUSTER_ID --region $REGION --steps "Type=Spark, Name='SystemDS Spark Program', ActionOnFailure=CONTINUE, Args=[ @@ -50,11 +50,11 @@ STEP_INFO=$(aws emr add-steps --cluster-id $CLUSTER_ID --steps "Type=Spark, --driver-memory,$SPARK_DRIVER_MEMORY, --num-executors,$SPARK_NUM_EXECUTORS, --conf,spark.driver.maxResultSize=0, - $SYSTEMDS_JAR_PATH, -f, s3://system-ds-bucket/$dml_filename, -exec, $SYSTEMDS_EXEC_MODE,$args,-stats, -explain]") + $SYSTEMDS_JAR_PATH, -f, s3://$BUCKET/$dml_filename, -exec, $SYSTEMDS_EXEC_MODE,$args,-stats, -explain]") STEP_ID=$(echo $STEP_INFO | jq .StepIds | tr -d '"' | tr -d ']' | tr -d '[' | tr -d '[:space:]' ) echo "Waiting for the step to finish" -aws emr wait step-complete --cluster-id $CLUSTER_ID --step-id $STEP_ID +aws emr wait step-complete --cluster-id $CLUSTER_ID --step-id $STEP_ID --region $REGION -aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stderr" -aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stdout" \ No newline at end of file +aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem --region $REGION --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stderr" +aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem --region $REGION --command "cat /mnt/var/log/hadoop/steps/$STEP_ID/stdout" \ No newline at end of file diff --git a/scripts/aws/spinup_systemds_cluster.sh b/scripts/aws/spinup_systemds_cluster.sh index 8c93a636eae..7200af05be3 100755 --- a/scripts/aws/spinup_systemds_cluster.sh +++ b/scripts/aws/spinup_systemds_cluster.sh @@ -46,9 +46,14 @@ set_config "SPARK_EXECUTOR_MEMORY" $SPARK_EXECUTOR_MEMORY set_config "SPARK_DRIVER_MEMORY" "1G" set_config "BUCKET" $BUCKET-$(((RANDOM % 999) + 1000)) +#Source again to update the changes for the current session +source systemds_cluster.config + #Create systemDS bucket -aws s3api create-bucket --bucket $BUCKET --region $REGION &> /dev/null -aws s3api create-bucket --bucket $BUCKET-logs --region $REGION &> /dev/null +#LocationConstraint configuration required regions outside of us-east-1 +if [ "$REGION" = "us-east-1" ]; then LOCATION_CONSTRAINT=""; else LOCATION_CONSTRAINT="--create-bucket-configuration LocationConstraint=$REGION"; fi +aws s3api create-bucket --bucket $BUCKET --region $REGION $LOCATION_CONSTRAINT &> /dev/null +aws s3api create-bucket --bucket $BUCKET-logs --region $REGION $LOCATION_CONSTRAINT &> /dev/null # Upload Jar and scripts to s3 aws s3 sync $SYSTEMDS_TARGET_DIRECTORY s3://$BUCKET --exclude "*" --include "*.dml" --include "*config.xml" --include "*DS.jar*" @@ -60,11 +65,17 @@ if [ ! -f ${KEYPAIR_NAME}.pem ]; then echo "${KEYPAIR_NAME}.pem private key created!" fi +#Get the first available subnet in the default VPC of the configured region +DEFAULT_SUBNET=$(aws ec2 describe-subnets --region $REGION \ + --filter "Name=defaultForAz,Values=true" --query "Subnets[0].SubnetId" --output text) + #Create the cluster +#Note: Ganglia not available since emr-6.15.0: exchanged with AmazonCloudWatchAgent CLUSTER_INFO=$(aws emr create-cluster \ - --applications Name=Ganglia Name=Spark \ + --applications Name=AmazonCloudWatchAgent Name=Spark \ --ec2-attributes '{"KeyName":"'${KEYPAIR_NAME}'", - "InstanceProfile":"EMR_EC2_DefaultRole"}'\ + "InstanceProfile":"EMR_EC2_DefaultRole", + "SubnetId": "'${DEFAULT_SUBNET}'"}'\ --service-role EMR_DefaultRole \ --enable-debugging \ --release-label $EMR_VERSION \ @@ -78,7 +89,13 @@ CLUSTER_INFO=$(aws emr create-cluster \ "InstanceGroupType":"CORE", "InstanceType":"'${INSTANCES_TYPE}'", "Name":"Core Instance Group"}]'\ - --configurations '[{"Classification":"spark","Properties":{"maximizeResourceAllocation": "true"}}]'\ + --configurations '[{"Classification":"spark","Properties":{"maximizeResourceAllocation": "true"}}, + {"Classification": "spark-env", + "Configurations": [{ + "Classification": "export", + "Properties": {"JAVA_HOME": "/usr/lib/jvm/jre-11"} + }] + }]'\ --scale-down-behavior TERMINATE_AT_TASK_COMPLETION \ --region $REGION) @@ -89,21 +106,22 @@ set_config "CLUSTER_ID" $CLUSTER_ID ip_address=$(curl ipecho.net/plain ; echo) #Add your ip to the security group -aws ec2 create-security-group --group-name ElasticMapReduce-master --description "info" &> /dev/null +aws ec2 create-security-group --group-name ElasticMapReduce-master --description "info" --region $REGION &> /dev/null aws ec2 authorize-security-group-ingress \ --group-name ElasticMapReduce-master \ --protocol tcp \ --port 22 \ - --cidr "${ip_address}"/24 &> /dev/null + --cidr "${ip_address}"/24 \ + --region $REGION &> /dev/null # Wait for cluster to start echo "Waiting for cluster running state" -aws emr wait cluster-running --cluster-id $CLUSTER_ID +aws emr wait cluster-running --cluster-id $CLUSTER_ID --region $REGION echo "Cluster info:" -export CLUSTER_URL=$(aws emr describe-cluster --cluster-id $CLUSTER_ID | jq .Cluster.MasterPublicDnsName | tr -d '"') +export CLUSTER_URL=$(aws emr describe-cluster --cluster-id $CLUSTER_ID --region $REGION | jq .Cluster.MasterPublicDnsName | tr -d '"') aws emr ssh --cluster-id $CLUSTER_ID --key-pair-file ${KEYPAIR_NAME}.pem --region $REGION \ - --command 'aws s3 cp s3://system-ds-bucket/target . --recursive --exclude "*" --include "*DS.jar*"' + --command 'aws s3 cp s3://'${BUCKET}' . --recursive --exclude "*" --include "*DS.jar*"' echo "Spinup finished." diff --git a/scripts/aws/systemds_cluster.config b/scripts/aws/systemds_cluster.config index a254bbc8649..8afed8d2bba 100644 --- a/scripts/aws/systemds_cluster.config +++ b/scripts/aws/systemds_cluster.config @@ -23,8 +23,8 @@ KEYPAIR_NAME="SystemDSkeynamex" REGION="us-east-1" -BUCKET="systemds-bucket" -EMR_VERSION="emr-5.28.0" +BUCKET=systemds-bucket +EMR_VERSION="emr-7.0.0" INSTANCES_TYPE="m5.xlarge" MASTER_INSTANCES_COUNT=1 diff --git a/scripts/aws/terminate_systemds_cluster.sh b/scripts/aws/terminate_systemds_cluster.sh index 2de546723c3..af4707612e4 100755 --- a/scripts/aws/terminate_systemds_cluster.sh +++ b/scripts/aws/terminate_systemds_cluster.sh @@ -22,10 +22,10 @@ source systemds_cluster.config -aws emr terminate-clusters --cluster-ids $CLUSTER_ID +aws emr terminate-clusters --cluster-ids $CLUSTER_ID --region $REGION # Wait for cluster to start echo "Waiting for cluster terminated state" -aws emr wait cluster-terminated --cluster-id $CLUSTER_ID +aws emr wait cluster-terminated --cluster-id $CLUSTER_ID --region $REGION echo "Cluster: ${CLUSTER_ID} terminated."