diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/pom.xml b/hadoop-shim-impls/hadoop-shim-2.7/pom.xml deleted file mode 100644 index 441c253310..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - hadoop-shim-impls - org.apache.tez - 1.0.0-SNAPSHOT - - 4.0.0 - hadoop-shim-2.7 - - - - org.slf4j - slf4j-api - - - junit - junit - - - org.apache.tez - hadoop-shim - - - org.apache.tez - hadoop-shim - test - test-jar - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-common - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java deleted file mode 100644 index bd1eabfa32..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.classification.InterfaceAudience.Private; - -@Private -public class HadoopShim25_26_27Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) { - return new HadoopShim27(); - } - return null; - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java b/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java deleted file mode 100644 index c5163036a1..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/java/org/apache/tez/hadoop/shim/HadoopShim27.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; - -@Private -public class HadoopShim27 extends HadoopShim { - - @Override - public void setHadoopCallerContext(String context) { - // Not supported - } - - @Override - public void clearHadoopCallerContext() { - // Not supported - } - - @Override - public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); - Set supportedTypes = new HashSet(); - for (SchedulerResourceTypes resourceType : supportedResourceTypes) { - supportedTypes.add(resourceType.name()); - } - return supportedTypes; - } - -} diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index 4226f1861e..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider diff --git a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java deleted file mode 100644 index c9937f8b94..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.7/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoop25_26_27ShimProvider { - - @Test - public void testShimProvider() { - HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 1)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 3)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 8)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7)); - - Assert.assertEquals(HadoopShim27.class, - provider.createHadoopShim("foo", 2, 7).getClass()); - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim27.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim25_26_27Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml deleted file mode 100644 index 5b11308f6d..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml deleted file mode 100644 index 58d4c1ea5e..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - hadoop-shim-impls - org.apache.tez - 1.0.0-SNAPSHOT - - 4.0.0 - hadoop-shim-2.8 - - - - org.slf4j - slf4j-api - - - junit - junit - - - org.apache.tez - hadoop-shim - - - org.apache.tez - hadoop-shim - test - test-jar - - - org.apache.hadoop - hadoop-yarn-api - - - org.apache.hadoop - hadoop-common - - - - diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java deleted file mode 100644 index c0d5577a4b..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -import org.apache.hadoop.ipc.CallerContext; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; - -public class HadoopShim28 extends HadoopShim { - - final static CallerContext nullCallerContext = new CallerContext.Builder("").build(); - - @Override - public void setHadoopCallerContext(String context) { - CallerContext.setCurrent(new CallerContext.Builder(context).build()); - } - - @Override - public void clearHadoopCallerContext() { - CallerContext.setCurrent(nullCallerContext); - } - - @Override - public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { - EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); - Set supportedTypes = new HashSet(); - for (SchedulerResourceTypes resourceType : supportedResourceTypes) { - supportedTypes.add(resourceType.name()); - } - return supportedTypes; - } - - @Override - public FinalApplicationStatus applyFinalApplicationStatusCorrection(FinalApplicationStatus orig, - boolean isSessionMode, boolean isError) { - switch (orig) { - case FAILED: - // App is failed if dag failed in non-session mode or there was an error. - return (!isSessionMode || isError) ? - FinalApplicationStatus.FAILED : FinalApplicationStatus.ENDED; - case SUCCEEDED: - return isSessionMode ? FinalApplicationStatus.ENDED : FinalApplicationStatus.SUCCEEDED; - default: - return orig; - } - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java deleted file mode 100644 index 318462d7e3..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -public class HadoopShim28Provider extends HadoopShimProvider { - - @Override - public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) { - if (majorVersion == 2 && minorVersion >= 8) { - return new HadoopShim28(); - } - return null; - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider deleted file mode 100644 index 24fe49d1f5..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider +++ /dev/null @@ -1,14 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -org.apache.tez.hadoop.shim.HadoopShim28Provider diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java deleted file mode 100644 index f3117c89dd..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; - -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoopShim28 { - - @Test - public void testApplyFinalApplicationStatusCorrection() { - HadoopShim shim = new HadoopShim28(); - // Session mode success/failure, change to ended - Assert.assertEquals(FinalApplicationStatus.ENDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, true, false)); - Assert.assertEquals(FinalApplicationStatus.ENDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, false)); - - // Non-session mode success/failure, retain success/failure - Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.SUCCEEDED, false, false)); - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, false)); - - // Session and non-session mode error, retain failed. - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, true, true)); - Assert.assertEquals(FinalApplicationStatus.FAILED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.FAILED, false, true)); - - // Session and non-session mode killed is killed. - Assert.assertEquals(FinalApplicationStatus.KILLED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, true, false)); - Assert.assertEquals(FinalApplicationStatus.KILLED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.KILLED, false, false)); - - // Session and non-session mode undefined is undefined. - Assert.assertEquals(FinalApplicationStatus.UNDEFINED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, true, false)); - Assert.assertEquals(FinalApplicationStatus.UNDEFINED, - shim.applyFinalApplicationStatusCorrection(FinalApplicationStatus.UNDEFINED, false, false)); - } -} diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java deleted file mode 100644 index f2f5e5ff71..0000000000 --- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tez.hadoop.shim; - -import org.apache.hadoop.conf.Configuration; - -import org.junit.Assert; -import org.junit.Test; - -public class TestHadoopShim28Provider { - - @Test - public void testShimProvider() { - HadoopShim28Provider provider = new HadoopShim28Provider(); - Assert.assertNull(provider.createHadoopShim("foo", 2, 2)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 7)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 5)); - Assert.assertNull(provider.createHadoopShim("foo", 2, 6)); - Assert.assertNull(provider.createHadoopShim("foo", 3, 3)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8)); - Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111)); - - Assert.assertEquals(HadoopShim28.class, - provider.createHadoopShim("foo", 2, 9).getClass()); - } - - @Test - public void testLoaderOverride() { - Configuration conf = new Configuration(false); - // Set shim and version to ensure that hadoop version from the build does not create - // a mismatch - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim28Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.8.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(HadoopShim28.class, shim.getClass()); - } - - @Test - public void testInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that shim in this module does not match - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.2.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - - @Test - public void testLoaderOverrideInvalidVersion() { - Configuration conf = new Configuration(false); - // Set incompatible version so that override shim does not return a valid shim - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, - HadoopShim28Provider.class.getName()); - conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0"); - HadoopShimsLoader loader = new HadoopShimsLoader(conf, true); - HadoopShim shim = loader.getHadoopShim(); - Assert.assertNotNull(shim); - Assert.assertEquals(DefaultHadoopShim.class, shim.getClass()); - } - -} diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml deleted file mode 100644 index fc3b2dcf8d..0000000000 --- a/hadoop-shim-impls/pom.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - 4.0.0 - - tez - org.apache.tez - 1.0.0-SNAPSHOT - - hadoop-shim-impls - pom - - - - hadoop27 - - false - - - hadoop-shim-2.7 - - - - hadoop28 - - true - - - hadoop-shim-2.8 - - - - - diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java index e7c7163e58..ef25b9326a 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java @@ -18,11 +18,38 @@ */ package org.apache.tez.hadoop.shim; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.ipc.CallerContext; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; -/** - * Default Hadoop Shim. For now, it mimics HadoopShim. - */ +/** Default Hadoop Shim. Provides Hadoop 3.x baseline capabilities. */ @Private public class DefaultHadoopShim extends HadoopShim { + + private static final CallerContext NULL_CALLER_CONTEXT = new CallerContext.Builder("").build(); + + @Override + public void setHadoopCallerContext(String context) { + CallerContext.setCurrent(new CallerContext.Builder(context).build()); + } + + @Override + public void clearHadoopCallerContext() { + CallerContext.setCurrent(NULL_CALLER_CONTEXT); + } + + @Override + public Set getSupportedResourceTypes(RegisterApplicationMasterResponse response) { + EnumSet supportedResourceTypes = response.getSchedulerResourceTypes(); + Set supportedTypes = new HashSet<>(); + for (SchedulerResourceTypes resourceType : supportedResourceTypes) { + supportedTypes.add(resourceType.name()); + } + return supportedTypes; + } } diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java index 96405f0ab4..56d642f330 100644 --- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java +++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java @@ -25,6 +25,16 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +/** + * An interoperability layer to handle API variations across different versions of Apache Hadoop. + * + *

NOTE TO DEVELOPERS: This shim framework dynamically loads version-specific implementations + * (via {@code HadoopShimsLoader}) at runtime. While the legacy Hadoop 2.x specific shims have been + * removed, the core framework is intentionally retained. It provides a generic, extensible + * mechanism to seamlessly handle any breaking API changes or missing features that may arise in + * future Hadoop versions, ensuring Tez maintains broad compatibility without tying the codebase to + * a single Hadoop API signature. + */ @Public @Unstable public abstract class HadoopShim { diff --git a/pom.xml b/pom.xml index 0aadd69e31..ccdc48d2d1 100644 --- a/pom.xml +++ b/pom.xml @@ -783,6 +783,11 @@ + + org.apache.hadoop + hadoop-hdfs-client + ${hadoop.version} + org.apache.hadoop hadoop-hdfs @@ -972,7 +977,6 @@ tez-ui tez-plugins tez-tools - hadoop-shim-impls tez-dist docs diff --git a/tez-api/pom.xml b/tez-api/pom.xml index 187e87d99b..5131258dcd 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -101,6 +101,15 @@ org.glassfish.jersey.media jersey-media-json-jackson + + org.apache.hadoop + hadoop-hdfs-client + + + org.apache.hadoop + hadoop-hdfs + test + org.apache.hadoop hadoop-hdfs @@ -221,38 +230,4 @@ - - - hadoop27 - - false - - - - org.apache.hadoop - hadoop-hdfs - runtime - - - - - hadoop28 - - true - - - - org.apache.hadoop - hadoop-hdfs-client - ${hadoop.version} - - - org.apache.hadoop - hadoop-hdfs - test - ${hadoop.version} - - - - diff --git a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java index 50af86039c..f8bc3e6101 100644 --- a/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java +++ b/tez-api/src/main/java/org/apache/tez/common/GuavaShim.java @@ -25,7 +25,14 @@ import com.google.common.util.concurrent.MoreExecutors; /** - * A interoperability layer to work with multiple versions of guava. + * An interoperability layer to work with multiple versions of guava. + * + *

NOTE TO DEVELOPERS: Reflection is used here to dynamically bind to the available executor + * method at runtime. Guava v18.0 deprecated and removed {@code sameThreadExecutor()}, replacing it + * with {@code directExecutor()}. By using reflection, Tez can remain safely compatible with both + * older and newer versions of Guava provided by the underlying Hadoop environment, without forcing + * a strict compile-time dependency on a specific Guava version. Do not replace this with a direct + * method call unless the minimum supported Guava version across all environments is >= 18.0. */ public final class GuavaShim { diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index e79950b0bf..806eb2a2fe 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -34,15 +34,15 @@ public class TestTimelineReaderFactory { - // ensure on hadoop 2.6+ TimelineReaderTokenAuthenticatedStrategy is used. + // ensure TimelineReaderTokenAuthenticatedStrategy is used. @Test(timeout = 5000) - public void testShouldUseTokenDelegationAuthStrategyForHadoop26() throws TezException { + public void testShouldUseTokenDelegationAuthStrategy() throws TezException { String returnedClassName = TimelineReaderFactory.getTimelineReaderStrategy(mock(Configuration.class), false, 0) .getClass() .getCanonicalName(); - Assert.assertEquals("should use pseudo auth on hadoop2.4", + Assert.assertEquals("should use token delegation auth", "org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderTokenAuthenticatedStrategy", returnedClassName); } diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml index 08a8077cca..5c7b0a3d12 100644 --- a/tez-dist/pom.xml +++ b/tez-dist/pom.xml @@ -58,6 +58,31 @@ ${project.version} provided + + org.apache.tez + tez-yarn-timeline-history-with-acls + ${project.version} + + + org.apache.tez + tez-yarn-timeline-history-with-fs + ${project.version} + + + org.apache.tez + tez-yarn-timeline-cache-plugin + ${project.version} + + + org.apache.tez + tez-protobuf-history-plugin + ${project.version} + + + org.apache.tez + hadoop-shim + ${project.version} + @@ -65,62 +90,6 @@ - - hadoop27 - - false - - - - org.apache.tez - tez-yarn-timeline-history-with-acls - ${project.version} - - - org.apache.tez - tez-protobuf-history-plugin - ${project.version} - - - org.apache.tez - hadoop-shim-2.7 - ${project.version} - - - - - hadoop28 - - true - - - - org.apache.tez - tez-yarn-timeline-history-with-acls - ${project.version} - - - org.apache.tez - tez-yarn-timeline-history-with-fs - ${project.version} - - - org.apache.tez - tez-yarn-timeline-cache-plugin - ${project.version} - - - org.apache.tez - tez-protobuf-history-plugin - ${project.version} - - - org.apache.tez - hadoop-shim-2.8 - ${project.version} - - - docker diff --git a/tez-plugins/pom.xml b/tez-plugins/pom.xml index 7001858ddb..88317f810e 100644 --- a/tez-plugins/pom.xml +++ b/tez-plugins/pom.xml @@ -29,37 +29,15 @@ tez-plugins pom - - - hadoop27 - - false - - - tez-protobuf-history-plugin - tez-yarn-timeline-history - tez-yarn-timeline-history-with-acls - tez-history-parser - tez-aux-services - - - - hadoop28 - - true - - - tez-protobuf-history-plugin - tez-yarn-timeline-history - tez-yarn-timeline-history-with-acls - tez-yarn-timeline-cache-plugin - tez-yarn-timeline-history-with-fs - tez-history-parser - tez-aux-services - - - - + + tez-protobuf-history-plugin + tez-yarn-timeline-history + tez-yarn-timeline-history-with-acls + tez-yarn-timeline-cache-plugin + tez-yarn-timeline-history-with-fs + tez-history-parser + tez-aux-services + diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index 5cd65b35bb..4df23c94f1 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -154,17 +154,4 @@ - - - hadoop27 - - - org.apache.tez - tez-yarn-timeline-history - ${project.version} - - - - - diff --git a/tez-tools/analyzers/pom.xml b/tez-tools/analyzers/pom.xml index aeda3a4631..cb971f398f 100644 --- a/tez-tools/analyzers/pom.xml +++ b/tez-tools/analyzers/pom.xml @@ -29,25 +29,8 @@ tez-perf-analyzer pom - - - hadoop27 - - false - - - job-analyzer - - - - hadoop28 - - true - - - job-analyzer - - - + + job-analyzer +