[Build] Refactor project structure and refactor release tools (#31)

Author: yuzelin

.github/workflows/check-java-bridge-licensing.yml

@@ -40,14 +40,14 @@ jobs:
       - name: Build
         run: |
           set -o pipefail
-          cd pypaimon/py4j/paimon-python-java-bridge
+          cd paimon-python-java-bridge
           mvn clean deploy ${{ env.MVN_COMMON_OPTIONS }} -DskipTests \
             -DaltDeploymentRepository=validation_repository::default::file:${{ env.MVN_VALIDATION_DIR }} \
             | tee ${{ env.MVN_BUILD_OUTPUT_FILE }}
 
       - name: Check licensing
         run: |
-          cd pypaimon/py4j/paimon-python-java-bridge
+          cd paimon-python-java-bridge
           mvn ${{ env.MVN_COMMON_OPTIONS }} exec:java@check-licensing -N \
             -Dexec.args="${{ env.MVN_BUILD_OUTPUT_FILE }} $(pwd) ${{ env.MVN_VALIDATION_DIR }}" \
             -Dlog4j.configurationFile=file://$(pwd)/tools/ci/log4j.properties

.github/workflows/create-source-release.yml

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Create Source Release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup GPG
+        env:
+          GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }}
+        run: |
+          mkdir -p ~/.gnupg
+          chmod 700 ~/.gnupg
+          echo $GPG_PRIVATE_KEY | base64 --decode | gpg --batch --import --yes
+          echo "use-agent" >> ~/.gnupg/gpg.conf
+          echo "pinentry-program /usr/bin/pinentry" >> ~/.gnupg/gpg-agent.conf
+          echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
+
+      - name: Create source release
+        env:
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+        run: |
+          mkdir -p output
+          chmod +x tools/releasing/create_source_release.sh
+          OUTPUT_DIR=output GPG_PASSPHRASE=$GPG_PASSPHRASE tools/releasing/create_source_release.sh          
+
+      - name: Upload source release
+        uses: actions/upload-artifact@v4
+        with:
+          name: source-release
+          path: output/*

.github/workflows/paimon-python-checks.yml

@@ -43,14 +43,8 @@ jobs:
         with:
           java-version: ${{ env.JDK_VERSION }}
           distribution: 'adopt'
-      - name: Set up hadoop dependency
-        run: |
-          mkdir -p ${{ github.workspace }}/temp
-          curl -L -o ${{ github.workspace }}/temp/bundled-hadoop.jar \
-          https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar
+
       - name: Run lint-python.sh
-        env:
-          _PYPAIMON_HADOOP_CLASSPATH: ${{ github.workspace }}/temp/bundled-hadoop.jar
         run: |
           chmod +x dev/lint-python.sh
           ./dev/lint-python.sh

MANIFEST.in

@@ -0,0 +1,23 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+global-exclude *.py[cod] __pycache__ .DS_Store
+recursive-include deps/jars *.jar
+include README.md
+include LICENSE
+include NOTICE

README.md

@@ -31,16 +31,6 @@ We provide script to check codes.
 ./dev/lint-python.sh -h # run this to see more usages         
 ```
 
-## Build
-
-We provide script to build wheel.
-
-```shell
-./dev/build-wheels.sh
-```
-
-The target wheel is under `dist/`
-
 # Usage
 
 See Apache Paimon Python API [Doc](https://paimon.apache.org/docs/master/program-api/python-api/).

dev/build-wheels.sh

@@ -577,8 +577,9 @@ function tox_check() {
     # Ensure the permission of the scripts set correctly
     chmod +x $PAIMON_PYTHON_DIR/dev/*
 
-    # tox runs codes in virtual env, set var to avoid error
-    export _PYPAIMON_TOX_TEST="true"
+    # dummy jar needed by setup.py
+    mkdir -p $PAIMON_PYTHON_DIR/deps/jars
+    touch $PAIMON_PYTHON_DIR/deps/jars/dummy.jar
 
     if [[ -n "$GITHUB_ACTION" ]]; then
         # Run tests in all versions triggered by a Git push (tests aren't so many currently)
@@ -596,6 +597,9 @@ function tox_check() {
         $TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini -e ${ENV_LIST[$index]} --recreate 2>&1 | tee -a $LOG_FILE
     fi
 
+    # delete dummy jar
+    rm -rf $PAIMON_PYTHON_DIR/deps
+
     TOX_RESULT=$((grep -c "congratulations :)" "$LOG_FILE") 2>&1)
     if [ $TOX_RESULT -eq '0' ]; then
         print_function "STAGE" "tox checks... [FAILED]"