ADH-8194, ADH-8212: Run tests SSM with hdfs 3.4.3

pom.xml

@@ -67,6 +67,7 @@
     <module>smart-web-server</module>
     <module>smart-hadoop-support</module>
     <module>smart-tests</module>
+    <module>smart-hive-exec</module>
     <module>smart-hive-support</module>
   </modules>
 
@@ -81,9 +82,11 @@
     <scala.version>2.11.12</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <mockito.version>2.28.2</mockito.version>
-    <testcontainers.version>1.20.4</testcontainers.version>
+    <testcontainers.version>1.21.4</testcontainers.version>
+    <docker-java.version>3.7.1</docker-java.version>
     <bouncycastle.version>1.68</bouncycastle.version>
     <protobuf.version>3.7.1</protobuf.version>
+    <protoc.artifact>com.google.protobuf:protoc:${protobuf.version}</protoc.artifact>
     <dbunitVersion>2.5.3</dbunitVersion>
     <akka.version>2.5.32</akka.version>
     <akka.config.version>1.2.1</akka.config.version>
@@ -109,7 +112,7 @@
     <spring.boot.version>2.7.18</spring.boot.version>
     <spring.boot.security.kerberos.version>1.0.1.RELEASE</spring.boot.security.kerberos.version>
     <openapi.generator.version>6.6.0</openapi.generator.version>
-    <commons.lang3.version>3.12.0</commons.lang3.version>
+    <commons.lang3.version>3.18.0</commons.lang3.version>
     <mapstruct.version>1.5.5.Final</mapstruct.version>
     <lombok-mapstruct-binding.version>0.2.0</lombok-mapstruct-binding.version>
     <aspectj.version>1.9.22</aspectj.version>
@@ -136,7 +139,8 @@
     <profile>
       <id>hadoop-3.3</id>
       <properties>
-        <hadoop.version>3.3.6</hadoop.version>
+        <hadoop.version>3.4.3</hadoop.version>
+        <kerby.version>2.0.3</kerby.version>
         <smart.hadoop.artifact>smart-hadoop-3.3</smart.hadoop.artifact>
       </properties>
       <activation>
@@ -604,6 +608,24 @@
         <version>${testcontainers.version}</version>
         <scope>test</scope>
       </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-api</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-transport</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>com.github.docker-java</groupId>
+        <artifactId>docker-java-transport-zerodep</artifactId>
+        <version>${docker-java.version}</version>
+        <scope>test</scope>
+      </dependency>
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>

smart-client/src/main/java/org/smartdata/client/SmartClient.java

@@ -17,7 +17,7 @@
  */
 package org.smartdata.client;
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.ProtobufRpcEngine2;
 import org.apache.hadoop.ipc.RPC;

smart-client/src/main/java/org/smartdata/client/SmartServerHandles.java

@@ -17,7 +17,7 @@
  */
 package org.smartdata.client;
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;

smart-common/pom.xml

@@ -148,7 +148,7 @@
                             <goal>run</goal>
                         </goals>
                         <configuration>
-                            <protocArtifact>com.google.protobuf:protoc:${protobuf.version}</protocArtifact>
+                            <protocArtifact>${protoc.artifact}</protocArtifact>
                             <protocVersion>${protobuf.version}</protocVersion>
                             <inputDirectories>
                                 <include>${basedir}/src/main/proto</include>
@@ -260,4 +260,19 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>apple-silicon-protoc</id>
+            <activation>
+                <os>
+                    <family>mac</family>
+                    <arch>aarch64</arch>
+                </os>
+            </activation>
+            <properties>
+                <protoc.artifact>com.google.protobuf:protoc:${protobuf.version}:exe:osx-x86_64</protoc.artifact>
+            </properties>
+        </profile>
+    </profiles>
 </project>

smart-common/src/main/java/org/smartdata/model/ActionInfo.java

@@ -21,7 +21,7 @@
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.Arrays;

smart-common/src/main/java/org/smartdata/model/RegexFilterSupport.java

@@ -18,7 +18,7 @@
 
 package org.smartdata.model;
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.Collection;
 import java.util.StringJoiner;

smart-common/src/main/java/org/smartdata/utils/ConfigUtil.java

@@ -18,7 +18,7 @@
 package org.smartdata.utils;
 
 import com.google.common.net.HostAndPort;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.smartdata.conf.SmartConfKeys;
 

smart-dist/pom.xml

@@ -38,12 +38,12 @@
         <!--        Image properties:-->
         <!--        ssm-hadoop-base image-->
         <hadoop.ssm.image.name>hub.adsw.io/ssm/ssm-hadoop-base</hadoop.ssm.image.name>
-        <hadoop.ssm.image.tag>3.3.6</hadoop.ssm.image.tag>
+        <hadoop.ssm.image.tag>3.4.3</hadoop.ssm.image.tag>
         <hadoop.ssm.image.file>
             target/docker/${hadoop.ssm.image.name}/${hadoop.ssm.image.tag}/build/docker-relative-dir/docker/multihost/Dockerfile-hadoop-base
         </hadoop.ssm.image.file>
         <hadoop.base.image.name>hub.adsw.io/java/hadoop-openjdk8</hadoop.base.image.name>
-        <hadoop.base.image.tag>3.3.6</hadoop.base.image.tag>
+        <hadoop.base.image.tag>3.4.3</hadoop.base.image.tag>
         <!--        ssm-server image-->
         <ssm.server.image.name>hub.adsw.io/ssm/ssm-server</ssm.server.image.name>
         <ssm.server.image.tag>${project.version}</ssm.server.image.tag>

smart-engine/src/main/java/org/smartdata/server/engine/CmdletManager.java

@@ -21,7 +21,7 @@
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ListMultimap;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.slf4j.Logger;

smart-hadoop-support/smart-hadoop/src/main/java/org/smartdata/hdfs/metric/fetcher/EventBatchSerializer.java

@@ -17,7 +17,7 @@
  */
 package org.smartdata.hdfs.metric.fetcher;
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.hdfs.inotify.EventBatch;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;

smart-hadoop-support/smart-hadoop/src/test/java/org/smartdata/hdfs/action/TestListFileAction.java

@@ -48,6 +48,7 @@ public class TestListFileAction extends MiniClusterHarness {
 
   private ListFileAction listFileAction;
   private TimeZone timeZoneToRestore;
+  private TimeZone fileDateFormatTimeZoneToRestore;
 
   @Parameter
   public boolean useFullRootPath;
@@ -60,7 +61,9 @@ public static Object[] parameters() {
   @Before
   public void buildAction() {
     timeZoneToRestore = TimeZone.getDefault();
+    fileDateFormatTimeZoneToRestore = ListFileAction.FILE_DATE_FORMAT.getTimeZone();
     TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
+    ListFileAction.FILE_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
 
     listFileAction = new ListFileAction();
     listFileAction.setLocalFileSystem(dfs);
@@ -112,6 +115,7 @@ public void createDirectoryForListing() {
   @After
   public void restoreTimezone() {
     TimeZone.setDefault(timeZoneToRestore);
+    ListFileAction.FILE_DATE_FORMAT.setTimeZone(fileDateFormatTimeZoneToRestore);
   }
 
   @Test

smart-hive-exec/pom.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.smartdata</groupId>
+        <artifactId>smartdata-project</artifactId>
+        <version>2.3.0-SNAPSHOT</version>
+        <relativePath>..</relativePath>
+    </parent>
+
+    <artifactId>smart-hive-exec</artifactId>
+    <version>2.3.0-SNAPSHOT</version>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-exec</artifactId>
+            <version>${hive.version}</version>
+            <optional>true</optional>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>${maven-shade-plugin.version}</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <artifactSet>
+                                <includes>
+                                    <include>org.apache.hive:hive-exec</include>
+                                </includes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <artifact>org.apache.hive:hive-exec</artifact>
+                                    <excludes>
+                                        <exclude>org/apache/commons/lang/**</exclude>
+                                        <exclude>org/apache/commons/lang3/**</exclude>
+                                        <exclude>META-INF/maven/commons-lang/commons-lang/**</exclude>
+                                        <exclude>META-INF/maven/org.apache.commons/commons-lang3/**</exclude>
+                                    </excludes>
+                                </filter>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

smart-hive-support/pom.xml

@@ -49,10 +49,16 @@
             <artifactId>lombok</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.smartdata</groupId>
+            <artifactId>smart-hive-exec</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-exec</artifactId>
             <version>${hive.version}</version>
+            <scope>provided</scope>
             <exclusions>
                 <exclusion>
                     <groupId>log4j</groupId>
@@ -151,4 +157,4 @@
         </dependency>
     </dependencies>
 
-</project>
+</project>

smart-hive-support/src/main/java/org/smartdata/hive/fetch/HmsInFlightEventSource.java

@@ -20,7 +20,7 @@
 import lombok.AccessLevel;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 import org.smartdata.hive.fetch.enrich.HmsEventEnricher;

smart-metastore/src/main/java/org/smartdata/metastore/MetaStore.java

@@ -17,7 +17,7 @@
  */
 package org.smartdata.metastore;
 
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.exception.NotFoundException;

smart-metastore/src/main/java/org/smartdata/metastore/ingestion/FileStatusIngester.java

@@ -20,7 +20,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.smartdata.metastore.MetaStore;
-import org.smartdata.metastore.MetaStoreException;
 import org.smartdata.model.FileInfo;
 import org.smartdata.model.FileInfoBatch;
 
@@ -40,30 +39,43 @@ public FileStatusIngester(MetaStore dbAdapter) {
 
   @Override
   public void run() {
+    IngestionTask.startPersisting();
     FileInfoBatch batch = IngestionTask.pollBatch();
     try {
-      if (batch != null) {
-        FileInfo[] statuses = batch.getFileInfos();
-        if (statuses.length == batch.actualSize()) {
-          this.dbAdapter.insertFiles(batch.getFileInfos(), true);
-          IngestionTask.numPersisted.addAndGet(statuses.length);
-        } else {
-          FileInfo[] actual = new FileInfo[batch.actualSize()];
-          System.arraycopy(statuses, 0, actual, 0, batch.actualSize());
-          this.dbAdapter.insertFiles(actual, true);
-          IngestionTask.numPersisted.addAndGet(actual.length);
-        }
+      if (batch == null) {
+        return;
+      }
 
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Consumer " + id + " " + batch.actualSize()
-              + " files insert into table 'files'.");
-        }
+      FileInfo[] statuses = batch.getFileInfos();
+      if (statuses.length == batch.actualSize()) {
+        this.dbAdapter.insertFiles(batch.getFileInfos(), true);
+        IngestionTask.numPersisted.addAndGet(statuses.length);
+      } else {
+        FileInfo[] actual = new FileInfo[batch.actualSize()];
+        System.arraycopy(statuses, 0, actual, 0, batch.actualSize());
+        this.dbAdapter.insertFiles(actual, true);
+        IngestionTask.numPersisted.addAndGet(actual.length);
       }
-    } catch (MetaStoreException e) {
-      // TODO: handle this issue
-      LOG.error("Consumer {} error", id);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Consumer " + id + " " + batch.actualSize()
+            + " files insert into table 'files'.");
+      }
+    } catch (Exception e) {
+      IngestionTask.fail(e);
+      if (batch == null) {
+        LOG.error("Consumer {} failed to poll a file status batch", id, e);
+      } else {
+        LOG.error("Consumer {} failed to persist {} file statuses",
+            id, batch.actualSize(), e);
+      }
+    } finally {
+      IngestionTask.finishPersisting();
+      logProgress();
     }
+  }
 
+  private void logProgress() {
     if (id == 0) {
       long curr = System.currentTimeMillis();
       if (curr - lastUpdateTime >= 5000) {