8344671: Few JFR streaming tests fail with application not alive error on MacOS 15

Backport-of: 12edcff1fcf19955de67d8d34e03f080ed905db6

Author: GoeLin

src/jdk.attach/macosx/classes/sun/tools/attach/VirtualMachineImpl.java

@@ -74,32 +74,35 @@ public class VirtualMachineImpl extends HotSpotVirtualMachine {
         if (!socket_file.exists()) {
             File f = createAttachFile(pid);
             try {
-                sendQuitTo(pid);
+                checkCatchesAndSendQuitTo(pid, false);
 
                 // give the target VM time to start the attach mechanism
                 final int delay_step = 100;
                 final long timeout = attachTimeout();
-                long time_spend = 0;
+                long time_spent = 0;
                 long delay = 0;
+
+                boolean timedout = false;
                 do {
                     // Increase timeout on each attempt to reduce polling
                     delay += delay_step;
                     try {
                         Thread.sleep(delay);
                     } catch (InterruptedException x) { }
 
-                    time_spend += delay;
-                    if (time_spend > timeout/2 && !socket_file.exists()) {
+                    timedout = (time_spent += delay) > timeout;
+
+                    if (time_spent > timeout/2 && !socket_file.exists()) {
                         // Send QUIT again to give target VM the last chance to react
-                        sendQuitTo(pid);
+                        checkCatchesAndSendQuitTo(pid, !timedout);
                     }
-                } while (time_spend <= timeout && !socket_file.exists());
+                } while (!timedout && !socket_file.exists());
                 if (!socket_file.exists()) {
                     throw new AttachNotSupportedException(
                         String.format("Unable to open socket file %s: " +
                                       "target process %d doesn't respond within %dms " +
                                       "or HotSpot VM not loaded", socket_path,
-                                      pid, time_spend));
+                                      pid, time_spent));
                 }
             } finally {
                 f.delete();
@@ -296,7 +299,7 @@ private File createAttachFile(int pid) throws IOException {
 
     //-- native methods
 
-    static native void sendQuitTo(int pid) throws IOException;
+    static native boolean checkCatchesAndSendQuitTo(int pid, boolean throwIfNotReady) throws IOException, AttachNotSupportedException;
 
     static native void checkPermissions(String path) throws IOException;
 

src/jdk.attach/macosx/native/libattach/VirtualMachineImpl.c

@@ -28,11 +28,13 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/syslimits.h>
+#include <sys/sysctl.h>
 #include <sys/types.h>
 #include <sys/un.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -116,15 +118,54 @@ JNIEXPORT void JNICALL Java_sun_tools_attach_VirtualMachineImpl_connect
 
 /*
  * Class:     sun_tools_attach_VirtualMachineImpl
- * Method:    sendQuitTo
+ * Method:    checkCatchesAndSendQuitTo
  * Signature: (I)V
  */
-JNIEXPORT void JNICALL Java_sun_tools_attach_VirtualMachineImpl_sendQuitTo
-  (JNIEnv *env, jclass cls, jint pid)
+JNIEXPORT jboolean JNICALL Java_sun_tools_attach_VirtualMachineImpl_checkCatchesAndSendQuitTo
+  (JNIEnv *env, jclass cls, jint pid, jboolean throwIfNotReady)
 {
-    if (kill((pid_t)pid, SIGQUIT)) {
-        JNU_ThrowIOExceptionWithLastError(env, "kill");
+    int mib[] = { CTL_KERN, KERN_PROC, KERN_PROC_PID, (int)pid };
+
+    struct kinfo_proc kiproc;
+    size_t            kipsz = sizeof(struct kinfo_proc);
+
+   /*
+    * Early in the lifetime of a JVM it has not yet initialized its signal handlers, in particular the QUIT
+    * handler, note that the default behavior of QUIT is to terminate the receiving process, if unhandled.
+    *
+    * Since we use QUIT to initiate an attach operation, if we signal a JVM during this period early in its
+    * lifetime before it has initialized its QUIT handler, such a signal delivery will terminate the JVM we
+    * are attempting to attach to!
+    *
+    * The following code guards the QUIT delivery by testing the current signal masks. It is okay to send QUIT
+    * if the signal is caught but not ignored, as that implies a handler has been installed.
+    */
+
+    if (sysctl(mib, sizeof(mib) / sizeof(int), &kiproc, &kipsz, NULL, 0) == 0) {
+        const bool ignored = (kiproc.kp_proc.p_sigignore & sigmask(SIGQUIT)) != 0;
+        const bool caught  = (kiproc.kp_proc.p_sigcatch & sigmask(SIGQUIT))  != 0;
+
+        // note: obviously the masks could change between testing and signalling however this is not the
+        // observed behavior of the current JVM implementation.
+
+        if (caught && !ignored) {
+            if (kill((pid_t)pid, SIGQUIT) != 0) {
+                JNU_ThrowIOExceptionWithLastError(env, "kill");
+            } else {
+                return JNI_TRUE;
+            }
+        } else if (throwIfNotReady) {
+            char msg[100];
+
+            snprintf(msg, sizeof(msg), "pid: %d, state is not ready to participate in attach handshake!", (int)pid);
+
+            JNU_ThrowByName(env, "com/sun/tools/attach/AttachNotSupportedException", msg);
+        }
+    } else {
+        JNU_ThrowIOExceptionWithLastError(env, "sysctl");
     }
+
+    return JNI_FALSE;
 }
 
 /*