diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml index cee916f3dd..95d1ed1795 100644 --- a/amoro-ams/pom.xml +++ b/amoro-ams/pom.xml @@ -316,6 +316,17 @@ s3-transfer-manager + + org.apache.hadoop + ${hadoop-client-api.artifact} + + + + org.apache.hadoop + ${hadoop-client-runtime.artifact} + runtime + + org.apache.hadoop hadoop-aws @@ -451,6 +462,12 @@ test + + junit + junit + test + + io.kubernetes client-java diff --git a/amoro-common/pom.xml b/amoro-common/pom.xml index c637dc198e..2b21ff9997 100644 --- a/amoro-common/pom.xml +++ b/amoro-common/pom.xml @@ -63,6 +63,21 @@ amoro-shade-zookeeper-3 + + commons-beanutils + commons-beanutils + + + + commons-codec + commons-codec + + + + commons-net + commons-net + + org.apache.commons commons-pool2 @@ -85,12 +100,12 @@ org.apache.hadoop - hadoop-auth + ${hadoop-client-api.artifact} org.apache.hadoop - hadoop-client + ${hadoop-client-runtime.artifact} diff --git a/amoro-format-iceberg/pom.xml b/amoro-format-iceberg/pom.xml index 50ea643aeb..2931d90d8a 100644 --- a/amoro-format-iceberg/pom.xml +++ b/amoro-format-iceberg/pom.xml @@ -129,12 +129,12 @@ org.apache.hadoop - hadoop-auth + ${hadoop-client-api.artifact} org.apache.hadoop - hadoop-client + ${hadoop-client-runtime.artifact} diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3 b/dev/deps/dependencies-hadoop-2-spark-3.3 index fd35cff138..f7da121e09 100644 --- a/dev/deps/dependencies-hadoop-2-spark-3.3 +++ b/dev/deps/dependencies-hadoop-2-spark-3.3 @@ -46,11 +46,11 @@ chill-java/0.10.0//chill-java-0.10.0.jar chill_2.12/0.10.0//chill_2.12-0.10.0.jar commons-beanutils/1.11.0//commons-beanutils-1.11.0.jar commons-cli/1.2//commons-cli-1.2.jar -commons-codec/1.15//commons-codec-1.15.jar +commons-codec/1.17.2//commons-codec-1.17.2.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar commons-compiler/3.0.16//commons-compiler-3.0.16.jar -commons-compress/1.22//commons-compress-1.22.jar +commons-compress/1.21//commons-compress-1.21.jar commons-configuration/1.6//commons-configuration-1.6.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar commons-dbcp/1.4//commons-dbcp-1.4.jar @@ -62,7 +62,7 @@ commons-lang/2.6//commons-lang-2.6.jar commons-lang3/3.14.0//commons-lang3-3.14.0.jar commons-logging/1.2//commons-logging-1.2.jar commons-math3/3.6.1//commons-math3-3.6.1.jar -commons-net/3.1//commons-net-3.1.jar +commons-net/3.12.0//commons-net-3.12.0.jar commons-pool/1.5.4//commons-pool-1.5.4.jar commons-pool2/2.12.0//commons-pool2-2.12.0.jar commons-text/1.10.0//commons-text-1.10.0.jar diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5 b/dev/deps/dependencies-hadoop-3-spark-3.5 index 1689f0ed63..b94f486ec5 100644 --- a/dev/deps/dependencies-hadoop-3-spark-3.5 +++ b/dev/deps/dependencies-hadoop-3-spark-3.5 @@ -3,7 +3,6 @@ RoaringBitmap/1.0.1//RoaringBitmap-1.0.1.jar ST4/4.3.1//ST4-4.3.1.jar activation/1.1.1//activation-1.1.1.jar aircompressor/0.27//aircompressor-0.27.jar -animal-sniffer-annotations/1.17//animal-sniffer-annotations-1.17.jar annotations/17.0.0//annotations-17.0.0.jar annotations/2.24.12//annotations-2.24.12.jar ant-launcher/1.9.1//ant-launcher-1.9.1.jar @@ -30,7 +29,6 @@ aws-core/2.24.12//aws-core-2.24.12.jar aws-json-protocol/2.24.12//aws-json-protocol-2.24.12.jar aws-query-protocol/2.24.12//aws-query-protocol-2.24.12.jar aws-xml-protocol/2.24.12//aws-xml-protocol-2.24.12.jar -bcprov-jdk15on/1.70//bcprov-jdk15on-1.70.jar bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar byte-buddy-agent/1.12.19//byte-buddy-agent-1.12.19.jar byte-buddy/1.12.19//byte-buddy-1.12.19.jar @@ -42,12 +40,11 @@ chill-java/0.10.0//chill-java-0.10.0.jar chill_2.12/0.10.0//chill_2.12-0.10.0.jar commons-beanutils/1.11.0//commons-beanutils-1.11.0.jar commons-cli/1.2//commons-cli-1.2.jar -commons-codec/1.16.1//commons-codec-1.16.1.jar +commons-codec/1.17.2//commons-codec-1.17.2.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-collections4/4.4//commons-collections4-4.4.jar commons-compiler/3.1.9//commons-compiler-3.1.9.jar commons-compress/1.23.0//commons-compress-1.23.0.jar -commons-configuration2/2.8.0//commons-configuration2-2.8.0.jar commons-crypto/1.1.0//commons-crypto-1.1.0.jar commons-dbcp/1.4//commons-dbcp-1.4.jar commons-dbcp2/2.9.0//commons-dbcp2-2.9.0.jar @@ -56,7 +53,7 @@ commons-lang/2.6//commons-lang-2.6.jar commons-lang3/3.14.0//commons-lang3-3.14.0.jar commons-logging/1.2//commons-logging-1.2.jar commons-math3/3.6.1//commons-math3-3.6.1.jar -commons-net/3.9.0//commons-net-3.9.0.jar +commons-net/3.12.0//commons-net-3.12.0.jar commons-pool/1.5.4//commons-pool-1.5.4.jar commons-pool2/2.12.0//commons-pool2-2.12.0.jar commons-text/1.10.0//commons-text-1.10.0.jar @@ -68,7 +65,6 @@ datanucleus-rdbms/4.1.19//datanucleus-rdbms-4.1.19.jar datasketches-java/3.3.0//datasketches-java-3.3.0.jar datasketches-memory/2.1.0//datasketches-memory-2.1.0.jar derby/10.14.2.0//derby-10.14.2.0.jar -dnsjava/3.4.0//dnsjava-3.4.0.jar dropwizard-metrics-hadoop-metrics2-reporter/0.1.2//dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar dynamodb/2.24.12//dynamodb-2.24.12.jar eclipse-collections-api/11.1.0//eclipse-collections-api-11.1.0.jar @@ -76,7 +72,6 @@ eclipse-collections/11.1.0//eclipse-collections-11.1.0.jar endpoints-spi/2.24.12//endpoints-spi-2.24.12.jar error_prone_annotations/2.10.0//error_prone_annotations-2.10.0.jar eventstream/1.0.1//eventstream-1.0.1.jar -failureaccess/1.0//failureaccess-1.0.jar flatbuffers-java/23.5.26//flatbuffers-java-23.5.26.jar flatbuffers/1.2.0-3f79e055//flatbuffers-1.2.0-3f79e055.jar flink-annotations/1.20.3//flink-annotations-1.20.3.jar @@ -102,21 +97,9 @@ flink-shaded-netty/4.1.91.Final-17.0//flink-shaded-netty-4.1.91.Final-17.0.jar flink-streaming-java/1.20.3//flink-streaming-java-1.20.3.jar glue/2.24.12//glue-2.24.12.jar gson/2.10.1//gson-2.10.1.jar -guava/27.0-jre//guava-27.0-jre.jar -hadoop-annotations/3.4.0//hadoop-annotations-3.4.0.jar -hadoop-auth/3.4.0//hadoop-auth-3.4.0.jar hadoop-aws/3.4.0//hadoop-aws-3.4.0.jar -hadoop-client/3.4.0//hadoop-client-3.4.0.jar -hadoop-common/3.4.0//hadoop-common-3.4.0.jar -hadoop-hdfs-client/3.4.0//hadoop-hdfs-client-3.4.0.jar -hadoop-mapreduce-client-common/3.4.0//hadoop-mapreduce-client-common-3.4.0.jar -hadoop-mapreduce-client-core/3.4.0//hadoop-mapreduce-client-core-3.4.0.jar -hadoop-mapreduce-client-jobclient/3.4.0//hadoop-mapreduce-client-jobclient-3.4.0.jar -hadoop-shaded-guava/1.2.0//hadoop-shaded-guava-1.2.0.jar -hadoop-shaded-protobuf_3_21/1.2.0//hadoop-shaded-protobuf_3_21-1.2.0.jar -hadoop-yarn-api/3.4.0//hadoop-yarn-api-3.4.0.jar -hadoop-yarn-client/3.4.0//hadoop-yarn-client-3.4.0.jar -hadoop-yarn-common/3.4.0//hadoop-yarn-common-3.4.0.jar +hadoop-client-api/3.4.0//hadoop-client-api-3.4.0.jar +hadoop-client-runtime/3.4.0//hadoop-client-runtime-3.4.0.jar hamcrest-core/1.3//hamcrest-core-1.3.jar hive-classification/3.1.3//hive-classification-3.1.3.jar hive-common/3.1.3//hive-common-3.1.3.jar @@ -160,23 +143,17 @@ iceberg-spark-extensions-3.5_2.12/1.6.1//iceberg-spark-extensions-3.5_2.12-1.6.1 icu4j/69.1//icu4j-69.1.jar identity-spi/2.24.12//identity-spi-2.24.12.jar ivy/2.5.1//ivy-2.5.1.jar -j2objc-annotations/1.1//j2objc-annotations-1.1.jar jackson-annotations/2.16.0//jackson-annotations-2.16.0.jar jackson-core/2.14.2//jackson-core-2.14.2.jar jackson-databind/2.14.2//jackson-databind-2.14.2.jar jackson-dataformat-yaml/2.17.1//jackson-dataformat-yaml-2.17.1.jar jackson-datatype-jsr310/2.17.1//jackson-datatype-jsr310-2.17.1.jar -jackson-jaxrs-base/2.12.7//jackson-jaxrs-base-2.12.7.jar -jackson-jaxrs-json-provider/2.12.7//jackson-jaxrs-json-provider-2.12.7.jar -jackson-module-jaxb-annotations/2.12.7//jackson-module-jaxb-annotations-2.12.7.jar jackson-module-scala_2.12/2.15.2//jackson-module-scala_2.12-2.15.2.jar -jakarta.activation-api/1.2.1//jakarta.activation-api-1.2.1.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.3//jakarta.servlet-api-4.0.3.jar jakarta.validation-api/2.0.2//jakarta.validation-api-2.0.2.jar jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar -jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar janino/3.1.9//janino-3.1.9.jar javalin/4.6.8//javalin-4.6.8.jar javassist/3.29.2-GA//javassist-3.29.2-GA.jar @@ -185,7 +162,6 @@ javax.jdo/3.2.0-m3//javax.jdo-3.2.0-m3.jar javax.json/1.0.4//javax.json-1.0.4.jar javax.servlet-api/3.1.0//javax.servlet-api-3.1.0.jar javolution/5.5.1//javolution-5.5.1.jar -jaxb-api/2.2.11//jaxb-api-2.2.11.jar jboss-logging/3.3.1.Final//jboss-logging-3.3.1.Final.jar jboss-threads/2.3.6.Final//jboss-threads-2.3.6.Final.jar jcip-annotations/1.0-1//jcip-annotations-1.0-1.jar @@ -215,7 +191,6 @@ json4s-ast_2.12/3.7.0-M11//json4s-ast_2.12-3.7.0-M11.jar json4s-core_2.12/3.7.0-M11//json4s-core_2.12-3.7.0-M11.jar json4s-jackson_2.12/3.7.0-M11//json4s-jackson_2.12-3.7.0-M11.jar json4s-scalap_2.12/3.7.0-M11//json4s-scalap_2.12-3.7.0-M11.jar -jsp-api/2.1//jsp-api-2.1.jar jsqlparser/4.7//jsqlparser-4.7.jar jsr305/3.0.0//jsr305-3.0.0.jar jta/1.1//jta-1.1.jar @@ -278,7 +253,6 @@ kyuubi-hive-jdbc-shaded/1.10.2//kyuubi-hive-jdbc-shaded-1.10.2.jar leveldbjni-all/1.8//leveldbjni-all-1.8.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.9.3//libthrift-0.9.3.jar -listenablefuture/9999.0-empty-to-avoid-conflict-with-guava//listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar log4j-api/2.20.0//log4j-api-2.20.0.jar log4j-core/2.20.0//log4j-core-2.20.0.jar @@ -323,7 +297,6 @@ netty-resolver-dns/4.1.128.Final//netty-resolver-dns-4.1.128.Final.jar netty-resolver/4.1.128.Final//netty-resolver-4.1.128.Final.jar netty-transport-classes-epoll/4.1.128.Final//netty-transport-classes-epoll-4.1.128.Final.jar netty-transport-classes-kqueue/4.1.128.Final//netty-transport-classes-kqueue-4.1.128.Final.jar -netty-transport-native-epoll/4.1.100.Final//netty-transport-native-epoll-4.1.100.Final.jar netty-transport-native-epoll/4.1.128.Final/linux-riscv64/netty-transport-native-epoll-4.1.128.Final-linux-riscv64.jar netty-transport-native-epoll/4.1.96.Final/linux-aarch_64/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar netty-transport-native-epoll/4.1.96.Final/linux-x86_64/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar @@ -334,7 +307,7 @@ netty-transport-rxtx/4.1.128.Final//netty-transport-rxtx-4.1.128.Final.jar netty-transport-sctp/4.1.128.Final//netty-transport-sctp-4.1.128.Final.jar netty-transport-udt/4.1.128.Final//netty-transport-udt-4.1.128.Final.jar netty-transport/4.1.128.Final//netty-transport-4.1.128.Final.jar -nimbus-jose-jwt/9.31//nimbus-jose-jwt-9.31.jar +nimbus-jose-jwt/9.30.1//nimbus-jose-jwt-9.30.1.jar objenesis/3.3//objenesis-3.3.jar okhttp/3.12.12//okhttp-3.12.12.jar okio/1.15.0//okio-1.15.0.jar @@ -360,11 +333,9 @@ profiles/2.24.12//profiles-2.24.12.jar protobuf-java/2.5.0//protobuf-java-2.5.0.jar protocol-core/2.24.12//protocol-core-2.24.12.jar py4j/0.10.9.7//py4j-0.10.9.7.jar -re2j/1.1//re2j-1.1.jar reactive-streams/1.0.4//reactive-streams-1.0.4.jar reflectasm/1.11.3//reflectasm-1.11.3.jar regions/2.24.12//regions-2.24.12.jar -reload4j/1.2.22//reload4j-1.2.22.jar rocksdbjni/7.10.2//rocksdbjni-7.10.2.jar s3-transfer-manager/2.24.12//s3-transfer-manager-2.24.12.jar s3/2.24.12//s3-2.24.12.jar @@ -398,7 +369,6 @@ spark-sql_2.12/3.5.7//spark-sql_2.12-3.5.7.jar spark-tags_2.12/3.5.7//spark-tags_2.12-3.5.7.jar spark-unsafe_2.12/3.5.7//spark-unsafe_2.12-3.5.7.jar sqlline/1.3.0//sqlline-1.3.0.jar -stax2-api/4.2.1//stax2-api-4.2.1.jar stream/2.9.6//stream-2.9.6.jar sts/2.24.12//sts-2.24.12.jar swagger-ui/5.17.14//swagger-ui-5.17.14.jar @@ -418,7 +388,6 @@ websocket-servlet/9.4.51.v20230217//websocket-servlet-9.4.51.v20230217.jar wildfly-client-config/1.0.1.Final//wildfly-client-config-1.0.1.Final.jar wildfly-common/1.5.4.Final//wildfly-common-1.5.4.Final.jar wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar -woodstox-core/5.4.0//woodstox-core-5.4.0.jar xbean-asm9-shaded/4.23//xbean-asm9-shaded-4.23.jar xnio-api/3.8.8.Final//xnio-api-3.8.8.Final.jar xz/1.9//xz-1.9.jar diff --git a/pom.xml b/pom.xml index 977595f7d5..b44c8a7abc 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,8 @@ 1.2.0 3.1.3 3.4.0 + hadoop-client-api + hadoop-client-runtime 2.0.3 2.12.15 2.12 @@ -114,9 +116,11 @@ 2.20.0 4.13.2 5.9.1 + 1.11.0 + 1.17.2 2.12.0 3.14.0 - 1.11.0 + 3.12.0 2.2.2 5.7.0 4.11.0 @@ -219,6 +223,24 @@ ${amoro-shade-thrift.version}-${amoro-shade.version} + + commons-beanutils + commons-beanutils + ${commons-beanutils.version} + + + + commons-codec + commons-codec + ${commons-codec.version} + + + + commons-net + commons-net + ${commons-net.version} + + org.apache.commons commons-pool2 @@ -316,6 +338,12 @@ hadoop-aliyun ${hadoop.version} ${aliyun-sdk-dependency-scope} + + + org.apache.hadoop + hadoop-common + + @@ -326,7 +354,7 @@ org.apache.hadoop - hadoop-auth + ${hadoop-client-runtime.artifact} ${hadoop.version} @@ -362,7 +390,7 @@ org.apache.hadoop - hadoop-client + ${hadoop-client-api.artifact} ${hadoop.version} @@ -452,6 +480,10 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-common + software.amazon.awssdk bundle @@ -892,6 +924,14 @@ hive-metastore ${hive.version} + + org.apache.curator + * + + + org.apache.hadoop + hadoop-common + org.apache.hadoop hadoop-hdfs @@ -1675,6 +1715,8 @@ 2.10.2 3.3.4 3.3 + hadoop-client + hadoop-auth