[HUDI-5251] Split GitHub actions CI by spark and flink (apache#7265)

ramiyer · Dec 15, 2022 · 972408b · 972408b
1 parent 6ef4772
commit 972408b
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 30 deletions.
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
@@ -13,34 +13,25 @@ env:
   MVN_ARGS: -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn
 
 jobs:
-  build:
+  test-spark:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         include:
           - scalaProfile: "scala-2.11"
             sparkProfile: "spark2.4"
-            flinkProfile: "flink1.13"
-
-          - scalaProfile: "scala-2.11"
-            sparkProfile: "spark2.4"
-            flinkProfile: "flink1.14"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark2.4"
-            flinkProfile: "flink1.13"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.1"
-            flinkProfile: "flink1.14"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.2"
-            flinkProfile: "flink1.14"
 
           - scalaProfile: "scala-2.12"
             sparkProfile: "spark3.3"
-            flinkProfile: "flink1.14"
 
     steps:
       - uses: actions/checkout@v2
@@ -54,38 +45,53 @@ jobs:
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
-          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn clean install -Pintegration-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DskipTests=true $MVN_ARGS
+          mvn clean install -Pintegration-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS
       - name: Quickstart Test
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
-          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark $MVN_ARGS
-      - name: Bundle Validation
+          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark $MVN_ARGS
+      - name: IT - Bundle Validation
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
-          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI
         run: |
           HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
           ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION
-      - name: Common Test
+      - name: UT - Common & Spark
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}
           SPARK_PROFILE: ${{ matrix.sparkProfile }}
-          FLINK_PROFILE: ${{ matrix.flinkProfile }}
         if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI
         run:
-          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" '-Dtest=Test*' -pl hudi-common $MVN_ARGS
-      - name: Spark SQL Test
+          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-common,hudi-spark-datasource/hudi-spark $MVN_ARGS
+
+  test-flink:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - flinkProfile: "flink1.13"
+          - flinkProfile: "flink1.14"
+          - flinkProfile: "flink1.15"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+          architecture: x64
+      - name: Build Project
+        env:
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+        run:
+          mvn clean install -Pintegration-tests -Dscala-2.12 -D"$FLINK_PROFILE" -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS
+      - name: Quickstart Test
         env:
-          SCALA_PROFILE: ${{ matrix.scalaProfile }}
-          SPARK_PROFILE: ${{ matrix.sparkProfile }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
-        if: ${{ !endsWith(env.SPARK_PROFILE, '2.4') }} # skip test spark 2.4 as it's covered by Azure CI
         run:
-          mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" '-Dtest=Test*' -pl hudi-spark-datasource/hudi-spark $MVN_ARGS
+          mvn test -Punit-tests -Dscala-2.12 -D"$FLINK_PROFILE" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink $MVN_ARGS
diff --git a/hudi-examples/hudi-examples-flink/pom.xml b/hudi-examples/hudi-examples-flink/pom.xml
@@ -204,8 +204,7 @@
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
-            <!-- Override the version to be same with Flink avro -->
-            <version>1.10.0</version>
+            <version>${flink.avro.version}</version>
             <scope>compile</scope>
         </dependency>
 

diff --git a/hudi-flink-datasource/hudi-flink/pom.xml b/hudi-flink-datasource/hudi-flink/pom.xml
@@ -230,8 +230,7 @@
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
-            <!-- Override the version to be same with Flink avro -->
-            <version>1.10.0</version>
+            <version>${flink.avro.version}</version>
             <scope>compile</scope>
         </dependency>
 

diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml
@@ -496,8 +496,7 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
-      <!-- Override the version to be same with Flink avro -->
-      <version>1.10.0</version>
+      <version>${flink.avro.version}</version>
       <scope>compile</scope>
     </dependency>
 

diff --git a/pom.xml b/pom.xml
@@ -137,6 +137,8 @@
     <flink.version>${flink1.15.version}</flink.version>
     <hudi.flink.module>hudi-flink1.15.x</hudi.flink.module>
     <flink.bundle.version>1.15</flink.bundle.version>
+    <!-- This is fixed to match with version from flink-avro -->
+    <flink.avro.version>1.10.0</flink.avro.version>
     <flink.format.parquet.version>1.12.2</flink.format.parquet.version>
     <flink.runtime.artifactId>flink-runtime</flink.runtime.artifactId>
     <flink.table.runtime.artifactId>flink-table-runtime</flink.table.runtime.artifactId>
@@ -2025,6 +2027,7 @@
       </modules>
       <properties>
         <skip.hudi-spark3.unit.tests>true</skip.hudi-spark3.unit.tests>
+        <avro.version>1.8.2</avro.version>
       </properties>
       <activation>
         <activeByDefault>true</activeByDefault>
@@ -2045,6 +2048,7 @@
       <properties>
         <sparkbundle.version>2.4</sparkbundle.version>
         <skip.hudi-spark3.unit.tests>true</skip.hudi-spark3.unit.tests>
+        <avro.version>1.8.2</avro.version>
       </properties>
       <activation>
         <property>