Skip to content

Closes #31. Add the ability to specify the view ACLs. Piggyback off of the MR config. #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xkrogen merged 1 commit into from
Apr 23, 2018
Merged

Closes #31. Add the ability to specify the view ACLs. Piggyback off of the MR config. #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions dynamometer-infra/src/main/java/com/linkedin/dynamometer/ApplicationMaster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
Expand Down Expand Up @@ -143,6 +144,8 @@ public class ApplicationMaster {
// Directory to use for remote storage (a location on the remote FS which
// can be accessed by all components)
private Path remoteStoragePath;
// The ACLs to view the launched containers
private Map<ApplicationAccessType, String> applicationAcls;
// The container the NameNode is running within
private volatile Container namenodeContainer;
// Map of the containers that the DataNodes are running within
Expand Down Expand Up @@ -210,6 +213,8 @@ public boolean init(String[] args) throws ParseException, IOException {
Map<String, String> envs = System.getenv();

remoteStoragePath = new Path(envs.get(DynoConstants.REMOTE_STORAGE_PATH_ENV));
applicationAcls = new HashMap<>();
applicationAcls.put(ApplicationAccessType.VIEW_APP, envs.get(DynoConstants.JOB_ACL_VIEW_ENV));
launchingUser = envs.get(Environment.USER.name());
if (envs.containsKey(DynoConstants.REMOTE_NN_RPC_ADDR_ENV)) {
launchNameNode = false;
Expand Down Expand Up @@ -653,6 +658,7 @@ public void run() {

// Set the environment
ctx.setEnvironment(amOptions.getShellEnv());
ctx.setApplicationACLs(applicationAcls);

try {
ctx.setLocalResources(getLocalResources());
Expand Down
8 changes: 8 additions & 0 deletions dynamometer-infra/src/main/java/com/linkedin/dynamometer/Client.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobStatus;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
Expand All @@ -61,6 +62,7 @@
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
Expand Down Expand Up @@ -438,6 +440,10 @@ public boolean run() throws IOException, YarnException {

// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
Map<ApplicationAccessType, String> acls = new HashMap<>();
acls.put(ApplicationAccessType.VIEW_APP, getConf().get(
MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));
amContainer.setApplicationACLs(acls);

FileSystem fs = FileSystem.get(getConf());
fs.mkdirs(getRemoteStoragePath(getConf(), infraAppId));
Expand Down Expand Up @@ -529,6 +535,8 @@ private Map<String, String> setupRemoteResourcesGetEnv() throws IOException {
setupRemoteResource(appMasterJar, infraAppId, DynoConstants.DYNO_JAR, env);

env.put(DynoConstants.BLOCK_LIST_PATH_ENV, blockListPath);
env.put(DynoConstants.JOB_ACL_VIEW_ENV,
getConf().get(MRJobConfig.JOB_ACL_VIEW_JOB, MRJobConfig.DEFAULT_JOB_ACL_VIEW_JOB));

env.put(DynoConstants.REMOTE_STORAGE_PATH_ENV, getRemoteStoragePath(getConf(), infraAppId).toString());

Expand Down
3 changes: 3 additions & 0 deletions dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ public boolean accept(Path path) {
// internally by this application
public static final String REMOTE_NN_RPC_ADDR_ENV = "REMOTE_NN_RPC_ADDR";

// Environment variable which will contain the view ACLs for the launched containers.
public static final String JOB_ACL_VIEW_ENV = "JOB_ACL_VIEW";

/* The following used for AM -> DN, NN communication */

// The name of the file which will store information about the NameNode
Expand Down