diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml
index cee916f3dd..95d1ed1795 100644
--- a/amoro-ams/pom.xml
+++ b/amoro-ams/pom.xml
@@ -316,6 +316,17 @@
s3-transfer-manager
+
+ org.apache.hadoop
+ ${hadoop-client-api.artifact}
+
+
+
+ org.apache.hadoop
+ ${hadoop-client-runtime.artifact}
+ runtime
+
+
org.apache.hadoop
hadoop-aws
@@ -451,6 +462,12 @@
test
+
+ junit
+ junit
+ test
+
+
io.kubernetes
client-java
diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml
index c637dc198e..2b21ff9997 100644
--- a/amoro-common/pom.xml
+++ b/amoro-common/pom.xml
@@ -63,6 +63,21 @@
amoro-shade-zookeeper-3
+
+ commons-beanutils
+ commons-beanutils
+
+
+
+ commons-codec
+ commons-codec
+
+
+
+ commons-net
+ commons-net
+
+
org.apache.commons
commons-pool2
@@ -85,12 +100,12 @@
org.apache.hadoop
- hadoop-auth
+ ${hadoop-client-api.artifact}
org.apache.hadoop
- hadoop-client
+ ${hadoop-client-runtime.artifact}
diff --git a/amoro-format-iceberg/pom.xml b/amoro-format-iceberg/pom.xml
index 50ea643aeb..2931d90d8a 100644
--- a/amoro-format-iceberg/pom.xml
+++ b/amoro-format-iceberg/pom.xml
@@ -129,12 +129,12 @@
org.apache.hadoop
- hadoop-auth
+ ${hadoop-client-api.artifact}
org.apache.hadoop
- hadoop-client
+ ${hadoop-client-runtime.artifact}
diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3 b/dev/deps/dependencies-hadoop-2-spark-3.3
index fd35cff138..f7da121e09 100644
--- a/dev/deps/dependencies-hadoop-2-spark-3.3
+++ b/dev/deps/dependencies-hadoop-2-spark-3.3
@@ -46,11 +46,11 @@ chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.12/0.10.0//chill_2.12-0.10.0.jar
commons-beanutils/1.11.0//commons-beanutils-1.11.0.jar
commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.15//commons-codec-1.15.jar
+commons-codec/1.17.2//commons-codec-1.17.2.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-collections4/4.4//commons-collections4-4.4.jar
commons-compiler/3.0.16//commons-compiler-3.0.16.jar
-commons-compress/1.22//commons-compress-1.22.jar
+commons-compress/1.21//commons-compress-1.21.jar
commons-configuration/1.6//commons-configuration-1.6.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
commons-dbcp/1.4//commons-dbcp-1.4.jar
@@ -62,7 +62,7 @@ commons-lang/2.6//commons-lang-2.6.jar
commons-lang3/3.14.0//commons-lang3-3.14.0.jar
commons-logging/1.2//commons-logging-1.2.jar
commons-math3/3.6.1//commons-math3-3.6.1.jar
-commons-net/3.1//commons-net-3.1.jar
+commons-net/3.12.0//commons-net-3.12.0.jar
commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-pool2/2.12.0//commons-pool2-2.12.0.jar
commons-text/1.10.0//commons-text-1.10.0.jar
diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5 b/dev/deps/dependencies-hadoop-3-spark-3.5
index 1689f0ed63..b94f486ec5 100644
--- a/dev/deps/dependencies-hadoop-3-spark-3.5
+++ b/dev/deps/dependencies-hadoop-3-spark-3.5
@@ -3,7 +3,6 @@ RoaringBitmap/1.0.1//RoaringBitmap-1.0.1.jar
ST4/4.3.1//ST4-4.3.1.jar
activation/1.1.1//activation-1.1.1.jar
aircompressor/0.27//aircompressor-0.27.jar
-animal-sniffer-annotations/1.17//animal-sniffer-annotations-1.17.jar
annotations/17.0.0//annotations-17.0.0.jar
annotations/2.24.12//annotations-2.24.12.jar
ant-launcher/1.9.1//ant-launcher-1.9.1.jar
@@ -30,7 +29,6 @@ aws-core/2.24.12//aws-core-2.24.12.jar
aws-json-protocol/2.24.12//aws-json-protocol-2.24.12.jar
aws-query-protocol/2.24.12//aws-query-protocol-2.24.12.jar
aws-xml-protocol/2.24.12//aws-xml-protocol-2.24.12.jar
-bcprov-jdk15on/1.70//bcprov-jdk15on-1.70.jar
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
byte-buddy-agent/1.12.19//byte-buddy-agent-1.12.19.jar
byte-buddy/1.12.19//byte-buddy-1.12.19.jar
@@ -42,12 +40,11 @@ chill-java/0.10.0//chill-java-0.10.0.jar
chill_2.12/0.10.0//chill_2.12-0.10.0.jar
commons-beanutils/1.11.0//commons-beanutils-1.11.0.jar
commons-cli/1.2//commons-cli-1.2.jar
-commons-codec/1.16.1//commons-codec-1.16.1.jar
+commons-codec/1.17.2//commons-codec-1.17.2.jar
commons-collections/3.2.2//commons-collections-3.2.2.jar
commons-collections4/4.4//commons-collections4-4.4.jar
commons-compiler/3.1.9//commons-compiler-3.1.9.jar
commons-compress/1.23.0//commons-compress-1.23.0.jar
-commons-configuration2/2.8.0//commons-configuration2-2.8.0.jar
commons-crypto/1.1.0//commons-crypto-1.1.0.jar
commons-dbcp/1.4//commons-dbcp-1.4.jar
commons-dbcp2/2.9.0//commons-dbcp2-2.9.0.jar
@@ -56,7 +53,7 @@ commons-lang/2.6//commons-lang-2.6.jar
commons-lang3/3.14.0//commons-lang3-3.14.0.jar
commons-logging/1.2//commons-logging-1.2.jar
commons-math3/3.6.1//commons-math3-3.6.1.jar
-commons-net/3.9.0//commons-net-3.9.0.jar
+commons-net/3.12.0//commons-net-3.12.0.jar
commons-pool/1.5.4//commons-pool-1.5.4.jar
commons-pool2/2.12.0//commons-pool2-2.12.0.jar
commons-text/1.10.0//commons-text-1.10.0.jar
@@ -68,7 +65,6 @@ datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar
datasketches-java/3.3.0//datasketches-java-3.3.0.jar
datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar
derby/10.14.2.0//derby-10.14.2.0.jar
-dnsjava/3.4.0//dnsjava-3.4.0.jar
dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar
dynamodb/2.24.12//dynamodb-2.24.12.jar
eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar
@@ -76,7 +72,6 @@ eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar
endpoints-spi/2.24.12//endpoints-spi-2.24.12.jar
error_prone_annotations/2.10.0//error_prone_annotations-2.10.0.jar
eventstream/1.0.1//eventstream-1.0.1.jar
-failureaccess/1.0//failureaccess-1.0.jar
flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar
flatbuffers/1.2.0-3f79e055//flatbuffers-1.2.0-3f79e055.jar
flink-annotations/1.20.3//flink-annotations-1.20.3.jar
@@ -102,21 +97,9 @@ flink-shaded-netty/4.1.91.Final-17.0//flink-shaded-netty-4.1.91.Final-17.0.jar
flink-streaming-java/1.20.3//flink-streaming-java-1.20.3.jar
glue/2.24.12//glue-2.24.12.jar
gson/2.10.1//gson-2.10.1.jar
-guava/27.0-jre//guava-27.0-jre.jar
-hadoop-annotations/3.4.0//hadoop-annotations-3.4.0.jar
-hadoop-auth/3.4.0//hadoop-auth-3.4.0.jar
hadoop-aws/3.4.0//hadoop-aws-3.4.0.jar
-hadoop-client/3.4.0//hadoop-client-3.4.0.jar
-hadoop-common/3.4.0//hadoop-common-3.4.0.jar
-hadoop-hdfs-client/3.4.0//hadoop-hdfs-client-3.4.0.jar
-hadoop-mapreduce-client-common/3.4.0//hadoop-mapreduce-client-common-3.4.0.jar
-hadoop-mapreduce-client-core/3.4.0//hadoop-mapreduce-client-core-3.4.0.jar
-hadoop-mapreduce-client-jobclient/3.4.0//hadoop-mapreduce-client-jobclient-3.4.0.jar
-hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar
-hadoop-shaded-protobuf_3_21/1.2.0//hadoop-shaded-protobuf_3_21-1.2.0.jar
-hadoop-yarn-api/3.4.0//hadoop-yarn-api-3.4.0.jar
-hadoop-yarn-client/3.4.0//hadoop-yarn-client-3.4.0.jar
-hadoop-yarn-common/3.4.0//hadoop-yarn-common-3.4.0.jar
+hadoop-client-api/3.4.0//hadoop-client-api-3.4.0.jar
+hadoop-client-runtime/3.4.0//hadoop-client-runtime-3.4.0.jar
hamcrest-core/1.3//hamcrest-core-1.3.jar
hive-classification/3.1.3//hive-classification-3.1.3.jar
hive-common/3.1.3//hive-common-3.1.3.jar
@@ -160,23 +143,17 @@ iceberg-spark-extensions-3.5_2.12/1.6.1//iceberg-spark-extensions-3.5_2.12-1.6.1
icu4j/69.1//icu4j-69.1.jar
identity-spi/2.24.12//identity-spi-2.24.12.jar
ivy/2.5.1//ivy-2.5.1.jar
-j2objc-annotations/1.1//j2objc-annotations-1.1.jar
jackson-annotations/2.16.0//jackson-annotations-2.16.0.jar
jackson-core/2.14.2//jackson-core-2.14.2.jar
jackson-databind/2.14.2//jackson-databind-2.14.2.jar
jackson-dataformat-yaml/2.17.1//jackson-dataformat-yaml-2.17.1.jar
jackson-datatype-jsr310/2.17.1//jackson-datatype-jsr310-2.17.1.jar
-jackson-jaxrs-base/2.12.7//jackson-jaxrs-base-2.12.7.jar
-jackson-jaxrs-json-provider/2.12.7//jackson-jaxrs-json-provider-2.12.7.jar
-jackson-module-jaxb-annotations/2.12.7//jackson-module-jaxb-annotations-2.12.7.jar
jackson-module-scala_2.12/2.15.2//jackson-module-scala_2.12-2.15.2.jar
-jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar
jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar
jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar
jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar
-jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar
janino/3.1.9//janino-3.1.9.jar
javalin/4.6.8//javalin-4.6.8.jar
javassist/3.29.2-GA//javassist-3.29.2-GA.jar
@@ -185,7 +162,6 @@ javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar
javax.json/1.0.4//javax.json-1.0.4.jar
javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar
javolution/5.5.1//javolution-5.5.1.jar
-jaxb-api/2.2.11//jaxb-api-2.2.11.jar
jboss-logging/3.3.1.Final//jboss-logging-3.3.1.Final.jar
jboss-threads/2.3.6.Final//jboss-threads-2.3.6.Final.jar
jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar
@@ -215,7 +191,6 @@ json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar
json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar
json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar
json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar
-jsp-api/2.1//jsp-api-2.1.jar
jsqlparser/4.7//jsqlparser-4.7.jar
jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
@@ -278,7 +253,6 @@ kyuubi-hive-jdbc-shaded/1.10.2//kyuubi-hive-jdbc-shaded-1.10.2.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
libthrift/0.9.3//libthrift-0.9.3.jar
-listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar
log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar
log4j-api/2.20.0//log4j-api-2.20.0.jar
log4j-core/2.20.0//log4j-core-2.20.0.jar
@@ -323,7 +297,6 @@ netty-resolver-dns/4.1.128.Final//netty-resolver-dns-4.1.128.Final.jar
netty-resolver/4.1.128.Final//netty-resolver-4.1.128.Final.jar
netty-transport-classes-epoll/4.1.128.Final//netty-transport-classes-epoll-4.1.128.Final.jar
netty-transport-classes-kqueue/4.1.128.Final//netty-transport-classes-kqueue-4.1.128.Final.jar
-netty-transport-native-epoll/4.1.100.Final//netty-transport-native-epoll-4.1.100.Final.jar
netty-transport-native-epoll/4.1.128.Final/linux-riscv64/netty-transport-native-epoll-4.1.128.Final-linux-riscv64.jar
netty-transport-native-epoll/4.1.96.Final/linux-aarch_64/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar
netty-transport-native-epoll/4.1.96.Final/linux-x86_64/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar
@@ -334,7 +307,7 @@ netty-transport-rxtx/4.1.128.Final//netty-transport-rxtx-4.1.128.Final.jar
netty-transport-sctp/4.1.128.Final//netty-transport-sctp-4.1.128.Final.jar
netty-transport-udt/4.1.128.Final//netty-transport-udt-4.1.128.Final.jar
netty-transport/4.1.128.Final//netty-transport-4.1.128.Final.jar
-nimbus-jose-jwt/9.31//nimbus-jose-jwt-9.31.jar
+nimbus-jose-jwt/9.30.1//nimbus-jose-jwt-9.30.1.jar
objenesis/3.3//objenesis-3.3.jar
okhttp/3.12.12//okhttp-3.12.12.jar
okio/1.15.0//okio-1.15.0.jar
@@ -360,11 +333,9 @@ profiles/2.24.12//profiles-2.24.12.jar
protobuf-java/2.5.0//protobuf-java-2.5.0.jar
protocol-core/2.24.12//protocol-core-2.24.12.jar
py4j/0.10.9.7//py4j-0.10.9.7.jar
-re2j/1.1//re2j-1.1.jar
reactive-streams/1.0.4//reactive-streams-1.0.4.jar
reflectasm/1.11.3//reflectasm-1.11.3.jar
regions/2.24.12//regions-2.24.12.jar
-reload4j/1.2.22//reload4j-1.2.22.jar
rocksdbjni/7.10.2//rocksdbjni-7.10.2.jar
s3-transfer-manager/2.24.12//s3-transfer-manager-2.24.12.jar
s3/2.24.12//s3-2.24.12.jar
@@ -398,7 +369,6 @@ spark-sql_2.12/3.5.7//spark-sql_2.12-3.5.7.jar
spark-tags_2.12/3.5.7//spark-tags_2.12-3.5.7.jar
spark-unsafe_2.12/3.5.7//spark-unsafe_2.12-3.5.7.jar
sqlline/1.3.0//sqlline-1.3.0.jar
-stax2-api/4.2.1//stax2-api-4.2.1.jar
stream/2.9.6//stream-2.9.6.jar
sts/2.24.12//sts-2.24.12.jar
swagger-ui/5.17.14//swagger-ui-5.17.14.jar
@@ -418,7 +388,6 @@ websocket-servlet/9.4.51.v20230217//websocket-servlet-9.4.51.v20230217.jar
wildfly-client-config/1.0.1.Final//wildfly-client-config-1.0.1.Final.jar
wildfly-common/1.5.4.Final//wildfly-common-1.5.4.Final.jar
wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar
-woodstox-core/5.4.0//woodstox-core-5.4.0.jar
xbean-asm9-shaded/4.23//xbean-asm9-shaded-4.23.jar
xnio-api/3.8.8.Final//xnio-api-3.8.8.Final.jar
xz/1.9//xz-1.9.jar
diff --git a/pom.xml b/pom.xml
index 977595f7d5..b44c8a7abc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,8 @@
1.2.0
3.1.3
3.4.0
+ hadoop-client-api
+ hadoop-client-runtime
2.0.3
2.12.15
2.12
@@ -114,9 +116,11 @@
2.20.0
4.13.2
5.9.1
+ 1.11.0
+ 1.17.2
2.12.0
3.14.0
- 1.11.0
+ 3.12.0
2.2.2
5.7.0
4.11.0
@@ -219,6 +223,24 @@
${amoro-shade-thrift.version}-${amoro-shade.version}
+
+ commons-beanutils
+ commons-beanutils
+ ${commons-beanutils.version}
+
+
+
+ commons-codec
+ commons-codec
+ ${commons-codec.version}
+
+
+
+ commons-net
+ commons-net
+ ${commons-net.version}
+
+
org.apache.commons
commons-pool2
@@ -316,6 +338,12 @@
hadoop-aliyun
${hadoop.version}
${aliyun-sdk-dependency-scope}
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
@@ -326,7 +354,7 @@
org.apache.hadoop
- hadoop-auth
+ ${hadoop-client-runtime.artifact}
${hadoop.version}
@@ -362,7 +390,7 @@
org.apache.hadoop
- hadoop-client
+ ${hadoop-client-api.artifact}
${hadoop.version}
@@ -452,6 +480,10 @@
${hadoop.version}
+
+ org.apache.hadoop
+ hadoop-common
+
software.amazon.awssdk
bundle
@@ -892,6 +924,14 @@
hive-metastore
${hive.version}
+
+ org.apache.curator
+ *
+
+
+ org.apache.hadoop
+ hadoop-common
+
org.apache.hadoop
hadoop-hdfs
@@ -1675,6 +1715,8 @@
2.10.2
3.3.4
3.3
+ hadoop-client
+ hadoop-auth