From 1349c36a930dc31c8820e71d79608e6447c0e813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Thu, 20 Feb 2025 16:06:31 +0800 Subject: [PATCH 1/6] enbale hadoop CallerContext in TezChild and DAGAppMaster --- .../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 4 +++- .../src/main/java/org/apache/tez/runtime/task/TezChild.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 4172a5a368..45b5266ff2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2390,7 +2390,9 @@ public static void main(String[] args) { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); - + org.apache.hadoop.ipc.CallerContext.setCurrent(new org.apache.hadoop.ipc.CallerContext + .Builder("tez_appmaster_" + containerId.getApplicationAttemptId() + ).build()); long appSubmitTime = Long.parseLong(appSubmitTimeStr); String jobUserName = System diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index ed14bd880c..4fed4c852b 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -533,6 +533,7 @@ public static void main(String[] args) throws IOException, InterruptedException, final int attemptNumber = Integer.parseInt(args[4]); final String[] localDirs = TezCommonUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS .name())); + CallerContext.setCurrent(new CallerContext.Builder("tez_"+tokenIdentifier).build()); LOG.info("TezChild starting with PID=" + pid + ", containerIdentifier=" + containerIdentifier); if (LOG.isDebugEnabled()) { LOG.debug("Info from cmd line: AM-host: " + host + " AM-port: " + port From 8fe1b40bc83902c7f5dc10133db75c973d66b467 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Thu, 20 Feb 2025 16:29:12 +0800 Subject: [PATCH 2/6] TEZ-4605:enbale hadoop CallerContext in TezChild and DAGAppMaster --- .../src/main/java/org/apache/tez/runtime/task/TezChild.java | 1 + 1 file changed, 1 insertion(+) diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java index 4fed4c852b..1e9ff61065 100644 --- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java +++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java @@ -41,6 +41,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; From 65d2ce4d5d9e871e16691611b416a2e280e5e191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Fri, 21 Feb 2025 09:45:51 +0800 Subject: [PATCH 3/6] use callercontext append rather than overwrite --- .../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 45b5266ff2..5180d6316d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2390,9 +2390,11 @@ public static void main(String[] args) { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); + org.apache.hadoop.ipc.CallerContext callerContext = org.apache.hadoop.ipc.CallerContext.getCurrent(); org.apache.hadoop.ipc.CallerContext.setCurrent(new org.apache.hadoop.ipc.CallerContext - .Builder("tez_appmaster_" + containerId.getApplicationAttemptId() - ).build()); + .Builder("tez_appmaster_" + containerId.getApplicationAttemptId()) + .append(containerIdStr) + .build()); long appSubmitTime = Long.parseLong(appSubmitTimeStr); String jobUserName = System From 8b104067ad8a66150b8100b910b96e4cf88dd627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Fri, 21 Feb 2025 11:24:20 +0800 Subject: [PATCH 4/6] use callercontext append rather than overwrite --- tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 5180d6316d..27eebb6abd 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2393,7 +2393,7 @@ public static void main(String[] args) { org.apache.hadoop.ipc.CallerContext callerContext = org.apache.hadoop.ipc.CallerContext.getCurrent(); org.apache.hadoop.ipc.CallerContext.setCurrent(new org.apache.hadoop.ipc.CallerContext .Builder("tez_appmaster_" + containerId.getApplicationAttemptId()) - .append(containerIdStr) + .append(callerContext.getContext()) .build()); long appSubmitTime = Long.parseLong(appSubmitTimeStr); From 39a8e80774e308e4a5e663f772dbea391f5fd349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Fri, 21 Feb 2025 21:55:23 +0800 Subject: [PATCH 5/6] Revert "use callercontext append rather than overwrite" This reverts commit 8b104067ad8a66150b8100b910b96e4cf88dd627. --- tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 27eebb6abd..5180d6316d 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2393,7 +2393,7 @@ public static void main(String[] args) { org.apache.hadoop.ipc.CallerContext callerContext = org.apache.hadoop.ipc.CallerContext.getCurrent(); org.apache.hadoop.ipc.CallerContext.setCurrent(new org.apache.hadoop.ipc.CallerContext .Builder("tez_appmaster_" + containerId.getApplicationAttemptId()) - .append(callerContext.getContext()) + .append(containerIdStr) .build()); long appSubmitTime = Long.parseLong(appSubmitTimeStr); From 908517a2dc1d37eb9380e54ea956f853329981d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BD=99=E8=89=AF?= Date: Fri, 21 Feb 2025 22:03:27 +0800 Subject: [PATCH 6/6] Revert "use callercontext append rather than overwrite" This reverts commit 65d2ce4d5d9e871e16691611b416a2e280e5e191. --- .../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java index 5180d6316d..45b5266ff2 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java @@ -2390,11 +2390,9 @@ public static void main(String[] args) { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId(); - org.apache.hadoop.ipc.CallerContext callerContext = org.apache.hadoop.ipc.CallerContext.getCurrent(); org.apache.hadoop.ipc.CallerContext.setCurrent(new org.apache.hadoop.ipc.CallerContext - .Builder("tez_appmaster_" + containerId.getApplicationAttemptId()) - .append(containerIdStr) - .build()); + .Builder("tez_appmaster_" + containerId.getApplicationAttemptId() + ).build()); long appSubmitTime = Long.parseLong(appSubmitTimeStr); String jobUserName = System