@@ -40,9 +40,12 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
4040 private var kubernetesTestComponents : KubernetesTestComponents = _
4141 private var sparkAppConf : SparkAppConf = _
4242
43- private val driverImage = System .getProperty(" spark.docker.test.driverImage" , " spark-driver:latest" )
44- private val executorImage = System .getProperty(" spark.docker.test.executorImage" , " spark-executor:latest" )
45- private val initContainerImage = System .getProperty(" spark.docker.test.initContainerImage" , " spark-init:latest" )
43+ private val driverImage = System .getProperty(" spark.docker.test.driverImage" ,
44+ " spark-driver:latest" )
45+ private val executorImage = System .getProperty(" spark.docker.test.executorImage" ,
46+ " spark-executor:latest" )
47+ private val initContainerImage = System .getProperty(" spark.docker.test.initContainerImage" ,
48+ " spark-init:latest" )
4649
4750
4851 override def beforeAll (): Unit = {
@@ -91,11 +94,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
9194 runSparkPiAndVerifyCompletion(appArgs = Array (" 5" ))
9295 }
9396
94- test(" Run SparkPi using the remote example jar." ) {
95- sparkAppConf.set(" spark.kubernetes.initContainer.image" , initContainerImage)
96- runSparkPiAndVerifyCompletion(appResource = REMOTE_EXAMPLES_JAR_URI )
97- }
98-
9997 test(" Run SparkPi with custom driver pod name, labels, annotations, and environment variables." ) {
10098 sparkAppConf
10199 .set(" spark.kubernetes.driver.pod.name" , " spark-integration-spark-pi" )
@@ -140,17 +138,30 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
140138 })
141139 }
142140
143- test(" Run SparkPi using the remote example jar with a test secret mounted into the driver and " +
144- " executor pods" ) {
141+ test(" Run PageRank using remote data file" ) {
145142 sparkAppConf
143+ .set(" spark.kubernetes.mountDependencies.filesDownloadDir" ,
144+ CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
145+ .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
146+ .set(" spark.kubernetes.initContainer.image" , initContainerImage)
147+ runSparkPageRankAndVerifyCompletion(
148+ appArgs = Array (CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ))
149+ }
150+
151+ test(" Run PageRank using remote data file with test secret mounted into the driver and " +
152+ " executors" ) {
153+ sparkAppConf
154+ .set(" spark.kubernetes.mountDependencies.filesDownloadDir" ,
155+ CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
156+ .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
146157 .set(s " spark.kubernetes.driver.secrets. $TEST_SECRET_NAME" , TEST_SECRET_MOUNT_PATH )
147158 .set(s " spark.kubernetes.executor.secrets. $TEST_SECRET_NAME" , TEST_SECRET_MOUNT_PATH )
148- sparkAppConf .set(" spark.kubernetes.initContainer.image" , initContainerImage)
159+ .set(" spark.kubernetes.initContainer.image" , initContainerImage)
149160
150161 createTestSecret()
151162
152- runSparkPiAndVerifyCompletion (
153- appResource = REMOTE_EXAMPLES_JAR_URI ,
163+ runSparkPageRankAndVerifyCompletion (
164+ appArgs = Array ( CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ) ,
154165 driverPodChecker = (driverPod : Pod ) => {
155166 doBasicDriverPodCheck(driverPod)
156167 checkTestSecret(driverPod, withInitContainer = true )
@@ -161,14 +172,6 @@ private[spark] class KubernetesSuite extends FunSuite with BeforeAndAfterAll wit
161172 })
162173 }
163174
164- test(" Run PageRank using remote data file" ) {
165- sparkAppConf
166- .set(" spark.kubernetes.mountDependencies.filesDownloadDir" , CONTAINER_LOCAL_FILE_DOWNLOAD_PATH )
167- .set(" spark.files" , REMOTE_PAGE_RANK_DATA_FILE )
168- .set(" spark.kubernetes.initContainer.image" , initContainerImage)
169- runSparkPageRankAndVerifyCompletion(appArgs = Array (CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE ))
170- }
171-
172175 private def runSparkPiAndVerifyCompletion (
173176 appResource : String = CONTAINER_LOCAL_SPARK_DISTRO_EXAMPLES_JAR ,
174177 driverPodChecker : Pod => Unit = doBasicDriverPodCheck,
@@ -322,9 +325,6 @@ private[spark] object KubernetesSuite {
322325
323326 val CONTAINER_LOCAL_FILE_DOWNLOAD_PATH = " /var/spark-data/spark-files"
324327
325- val REMOTE_EXAMPLES_JAR_URI =
326- " https://storage.googleapis.com/spark-k8s-integration-tests/jars/spark-examples_2.11-2.3.0.jar"
327-
328328 val REMOTE_PAGE_RANK_DATA_FILE =
329329 " https://storage.googleapis.com/spark-k8s-integration-tests/files/pagerank_data.txt"
330330 val CONTAINER_LOCAL_DOWNLOADED_PAGE_RANK_DATA_FILE =
0 commit comments