Skip to content

Commit e51fcfb

Browse files
sayedbilalbarisayedbilalbari
committed
Reverting NDS related changes
Signed-off-by: Sayed Bilal Bari <[email protected]>
1 parent fc11bba commit e51fcfb

File tree

5 files changed

+218
-0
lines changed

5 files changed

+218
-0
lines changed

‎nds/jvm_listener/pom.xml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License");
6+
~ you may not use this file except in compliance with the License.
7+
~ You may obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
~ See the License for the specific language governing permissions and
15+
~ limitations under the License.
16+
-->
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
21+
<groupId>com.nvidia</groupId>
22+
<artifactId>nds-benchmark-listener</artifactId>
23+
<packaging>jar</packaging>
24+
<version>1.0-SNAPSHOT</version>
25+
26+
<properties>
27+
<maven.compiler.source>8</maven.compiler.source>
28+
<maven.compiler.target>8</maven.compiler.target>
29+
</properties>
30+
<dependencies>
31+
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core -->
32+
<dependency>
33+
<groupId>org.apache.spark</groupId>
34+
<artifactId>spark-core_2.12</artifactId>
35+
<version>3.1.2</version>
36+
</dependency>
37+
<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql -->
38+
<dependency>
39+
<groupId>org.apache.spark</groupId>
40+
<artifactId>spark-sql_2.12</artifactId>
41+
<version>3.1.2</version>
42+
<scope>provided</scope>
43+
</dependency>
44+
</dependencies>
45+
46+
<build>
47+
<sourceDirectory>src/main/scala/</sourceDirectory>
48+
<plugins>
49+
<plugin>
50+
<groupId>org.apache.maven.plugins</groupId>
51+
<artifactId>maven-compiler-plugin</artifactId>
52+
<version>3.8.1</version>
53+
<configuration>
54+
<source>1.8</source>
55+
<target>1.8</target>
56+
</configuration>
57+
</plugin>
58+
<plugin>
59+
<groupId>org.scala-tools</groupId>
60+
<artifactId>maven-scala-plugin</artifactId>
61+
<version>2.15.2</version>
62+
<executions>
63+
<execution>
64+
<goals>
65+
<goal>compile</goal>
66+
<goal>testCompile</goal>
67+
</goals>
68+
</execution>
69+
</executions>
70+
</plugin>
71+
</plugins>
72+
</build>
73+
74+
</project>

‎nds/jvm_listener/src/main/scala/com/nvidia/spark/rapids/listener/Listener.scala‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.nvidia.spark.rapids.listener
19+
20+
trait Listener {
21+
/* Listener interface to be implemented at Python side
22+
*/
23+
def notify(x: Any): Any
24+
}

‎nds/jvm_listener/src/main/scala/com/nvidia/spark/rapids/listener/Manager.scala‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.nvidia.spark.rapids.listener
19+
20+
import org.apache.spark.SparkContext
21+
22+
object Manager {
23+
/* Manager class to manage all extra customized listeners.
24+
*/
25+
private var listeners: Map[String, Listener] = Map()
26+
private val spark_listener = new TaskFailureListener()
27+
private var isRegistered = false
28+
29+
def register(listener: Listener): String = {
30+
/* Note this register method has nothing to do with SparkContext.addSparkListener method.
31+
* This method is only to provide an interface to developers to have a better control over
32+
* all customized listeners.
33+
*/
34+
this.synchronized {
35+
// We register to the spark listener when the first listener is registered.
36+
registerSparkListener()
37+
val uuid = java.util.UUID.randomUUID().toString
38+
listeners = listeners + (uuid -> listener)
39+
uuid
40+
}
41+
}
42+
43+
def unregister(uuid: String) = {
44+
this.synchronized {
45+
listeners = listeners - uuid
46+
}
47+
}
48+
49+
def notifyAll(message: String): Unit = {
50+
for { (_, listener) <- listeners } listener.notify(message)
51+
}
52+
53+
def registerSparkListener() : Unit = {
54+
if (!isRegistered) {
55+
SparkContext.getOrCreate().addSparkListener(spark_listener)
56+
isRegistered = true
57+
}
58+
}
59+
60+
def unregisterSparkListener() : Unit = {
61+
if (isRegistered) {
62+
SparkContext.getOrCreate().removeSparkListener(spark_listener)
63+
isRegistered = false
64+
}
65+
}
66+
}

‎nds/jvm_listener/src/main/scala/com/nvidia/spark/rapids/listener/TaskFailureListener.scala‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.nvidia.spark.rapids.listener
19+
20+
import org.apache.spark.{Success, TaskEndReason}
21+
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
22+
import scala.collection.mutable.ListBuffer
23+
24+
25+
/* A simple listener which captures SparkListenerTaskEnd,
26+
* extracts "reason" of the task. If the reason is not "Success",
27+
* send this reason to python side.
28+
*/
29+
class TaskFailureListener extends SparkListener {
30+
override def onTaskEnd(taskEnd: SparkListenerTaskEnd) {
31+
taskEnd.reason match {
32+
case Success =>
33+
case reason => Manager.notifyAll(reason.toString)
34+
}
35+
super.onTaskEnd(taskEnd)
36+
}
37+
}

‎nds/properties/aqe-on.properties‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#
2+
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
spark.sql.adaptive.enabled=true

0 commit comments

Comments
 (0)