[GR-70638] Add time tracker for benchmarking.

PullRequest: mx/1989

Author: farquet

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.68.2",
+  "mx_version": "7.68.4",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {
@@ -49,12 +49,12 @@
     "graalvm-ee-25-ea":   {"name": "graalvm-jdk", "version": "25.0.0", "ea": "36", "platformspecific": true },
 
     "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "25",      "build_id": "jdk-25.0.1+8", "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10", "platformspecific": true },
-    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10-debug", "platformspecific": true },
-    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10-sulong", "platformspecific": true },
-    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10", "platformspecific": true },
-    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10-debug", "platformspecific": true },
-    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10-sulong", "platformspecific": true }
+    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11", "platformspecific": true },
+    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11-debug", "platformspecific": true },
+    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11-sulong", "platformspecific": true },
+    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11", "platformspecific": true },
+    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11-debug", "platformspecific": true },
+    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11-sulong", "platformspecific": true }
   },
 
   "eclipse": {

src/mx/_impl/mx_benchmark.py

@@ -102,12 +102,13 @@
     "enable_tracker",
     "disable_tracker",
     "Tracker",
-    "RssTracker",
+    "TimeTracker",
     "PsrecordTracker",
     "PsrecordMaxrssTracker",
     "RssPercentilesTracker",
-    "RssPercentilesAndMaxTracker",
+    "RssPercentilesAndTimeTracker",
     "EnergyConsumptionTracker",
+    "get_tracker_class",
     "BenchmarkExecutor",
     "make_hwloc_bind",
     "init_benchmark_suites",
@@ -766,14 +767,36 @@ def register_command_mapper_hook(self, name: str, hook: Callable | MapperHook):
         else:
             raise ValueError(f"Hook must be MapperHook or callable, got {type(hook)}")
 
-
     def register_tracker(self, name, tracker_type):
         tracker = tracker_type(self)
         self._tracker = tracker
         hook = tracker.get_hook()
 
         self.register_command_mapper_hook(name, hook)
 
+    def unregister_tracker(self):
+        if self._tracker is None:
+            mx.warn("No tracker is currently registered to unregister")
+            return
+
+        tracker_type = type(self._tracker)
+        hook_name_to_remove = None
+
+        for name, registered_type in _available_trackers.items():
+            if registered_type == tracker_type:
+                hook_name_to_remove = name
+                break
+
+        if hook_name_to_remove and hook_name_to_remove in self._command_mapper_hooks:
+            del self._command_mapper_hooks[hook_name_to_remove]
+        else:
+            mx.warn("Could not find tracker's hook in registered command mapper hooks")
+
+        self._tracker = None
+
+    def tracker(self):
+        return self._tracker
+
     def version(self):
         """The suite version selected for execution which is either the :defaultSuiteVerion:
         or the :desiredVersion: if any.
@@ -3575,77 +3598,61 @@ def get_rules(self, bmSuiteArgs):
     def get_hook(self) -> MapperHook:
         return DefaultTrackerHook(self)
 
-class RssTracker(Tracker):
+class TimeTracker(Tracker):
     def __init__(self, bmSuite):
         super().__init__(bmSuite)
-        log_deprecation("The 'rss' tracker is deprecated, use 'rsspercentiles' instead!")
+        self._timing_wrapper = None
+
+        if mx.get_os() != "linux":
+            script_dir = os.path.dirname(os.path.abspath(__file__))
+            self._timing_wrapper = os.path.join(script_dir, "timing_wrapper.py")
+            if not os.path.exists(self._timing_wrapper):
+                mx.warn(f"Timing wrapper not found at {self._timing_wrapper}")
+                self._timing_wrapper = None
 
     def map_command(self, cmd):
         """
-        Tracks the max resident memory size used by the process using the 'time' command.
+        Tracks wall-clock time using platform-appropriate timing commands.
 
         :param list[str] cmd: the input command to modify
-        :return:
+        :return: list[str] modified command with timing prefix
         """
         if not _use_tracker:
             return cmd
+
         if mx.get_os() == "linux":
-            prefix = ["time", "-v"]
-        elif mx.get_os() == "darwin":
-            prefix = ["time", "-l"]
+            # Forces GNU time with '/usr/bin/time' instead of shell builtin with 'time'
+            prefix = ["/usr/bin/time", "-f", "Wall-clock time: %e sec"]
         else:
-            mx.log(f"Ignoring the 'rss' tracker since it is not supported on {mx.get_os()}")
-            prefix = []
+            if self._timing_wrapper and os.path.exists(self._timing_wrapper):
+                prefix = [self._timing_wrapper]
+            else:
+                raise ValueError(f"Timing wrapper not found at {self._timing_wrapper}")
+
         return prefix + cmd
 
     def get_rules(self, bmSuiteArgs):
         if mx_benchmark_compatibility().bench_suite_needs_suite_args():
             suite_name = self.bmSuite.benchSuiteName(bmSuiteArgs)
         else:
             suite_name = self.bmSuite.benchSuiteName()
-        if mx.get_os() == "linux":
-            # Output of 'time -v' on linux contains:
-            #        Maximum resident set size (kbytes): 511336
-            rules = [
-                StdOutRule(
-                    r"Maximum resident set size \(kbytes\): (?P<rss>[0-9]+)",
-                    {
-                        "benchmark": self.bmSuite.currently_running_benchmark(),
-                        "bench-suite": suite_name,
-                        "config.vm-flags": ' '.join(self.bmSuite.vmArgs(bmSuiteArgs)),
-                        "metric.name": "max-rss",
-                        "metric.value": ("<rss>", lambda x: int(float(x)/(1024))),
-                        "metric.unit": "MB",
-                        "metric.type": "numeric",
-                        "metric.score-function": "id",
-                        "metric.better": "lower",
-                        "metric.iteration": 0
-                    }
-                )
-            ]
-        elif mx.get_os() == "darwin":
-            # Output of 'time -l' on linux contains (size in bytes):
-            #  523608064  maximum resident set size
-            rules = [
-                StdOutRule(
-                    r"(?P<rss>[0-9]+)\s+maximum resident set size",
-                    {
-                        "benchmark": self.bmSuite.currently_running_benchmark(),
-                        "bench-suite": suite_name,
-                        "config.vm-flags": ' '.join(self.bmSuite.vmArgs(bmSuiteArgs)),
-                        "metric.name": "max-rss",
-                        "metric.value": ("<rss>", lambda x: int(float(x)/(1024*1024))),
-                        "metric.unit": "MB",
-                        "metric.type": "numeric",
-                        "metric.score-function": "id",
-                        "metric.better": "lower",
-                        "metric.iteration": 0
-                    }
-                )
-            ]
-        else:
-            rules = []
-        return rules
+        return [
+            StdOutRule(
+                r"Wall-clock time:\s+(?P<time_sec>[0-9]+\.[0-9]+) sec",
+                {
+                    "benchmark": self.bmSuite.currently_running_benchmark(),
+                    "bench-suite": suite_name,
+                    "config.vm-flags": ' '.join(self.bmSuite.vmArgs(bmSuiteArgs)),
+                    "metric.name": "time",
+                    "metric.unit": "ms",
+                    "metric.value": ("<time_sec>", lambda ms: float(ms) * 1000),
+                    "metric.type": "numeric",
+                    "metric.score-function": "id",
+                    "metric.better": "lower",
+                    "metric.iteration": 0
+                }
+            )
+        ]
 
 
 class PsrecordTracker(Tracker):
@@ -3727,7 +3734,7 @@ class PsrecordMaxrssTracker(Tracker):
 
     def __init__(self, bmSuite):
         super().__init__(bmSuite)
-        self.rss = RssTracker(bmSuite)
+        self.rss = TimeTracker(bmSuite)
         self.psrecord = PsrecordTracker(bmSuite)
 
     def map_command(self, cmd):
@@ -3911,22 +3918,21 @@ def parseResults(self, text):
             return []
 
 
-class RssPercentilesAndMaxTracker(Tracker):
+class RssPercentilesAndTimeTracker(Tracker):
     def __init__(self, bmSuite):
         super().__init__(bmSuite)
-        self.rss_max_tracker = RssTracker(bmSuite)
+        self.time_tracker = TimeTracker(bmSuite)
         self.rss_percentiles_tracker = RssPercentilesTracker(
             bmSuite,
             skip=1,                 # skip RSS of the 'time' command
-            copy_into_max_rss=False # don't copy a percentile into max-rss, since the rss_max_tracker will generate the metric
+            copy_into_max_rss=True
         )
-        log_deprecation("The 'rsspercentiles+maxrss' tracker is deprecated, use 'rsspercentiles' instead!")
 
     def map_command(self, cmd):
-        return self.rss_percentiles_tracker.map_command(self.rss_max_tracker.map_command(cmd))
+        return self.rss_percentiles_tracker.map_command(self.time_tracker.map_command(cmd))
 
     def get_rules(self, bmSuiteArgs):
-        return self.rss_max_tracker.get_rules(bmSuiteArgs) + self.rss_percentiles_tracker.get_rules(bmSuiteArgs)
+        return self.time_tracker.get_rules(bmSuiteArgs) + self.rss_percentiles_tracker.get_rules(bmSuiteArgs)
 
 class EnergyConsumptionTracker(Tracker):
     """
@@ -4060,14 +4066,19 @@ def should_apply(self, stage: Optional[Stage]) -> bool:
         return stage.is_final() if stage else False
 
 _available_trackers = {
-    "rss": RssTracker,
+    "time": TimeTracker,
     "psrecord": PsrecordTracker,
     "psrecord+maxrss": PsrecordMaxrssTracker,
     "rsspercentiles": RssPercentilesTracker,
-    "rsspercentiles+maxrss": RssPercentilesAndMaxTracker,
+    "rsspercentiles+time": RssPercentilesAndTimeTracker,
     "energy": EnergyConsumptionTracker
 }
 
+def get_tracker_class(tracker_name):
+    if tracker_name not in _available_trackers:
+        raise ValueError(f"Tracker '{tracker_name}' doesn't exist! Use one of: [{', '.join(_available_trackers.keys())}]")
+    return _available_trackers[tracker_name]
+
 
 class BenchmarkExecutor(object):
     def uid(self):

src/mx/_impl/timing_wrapper.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+#
+# ----------------------------------------------------------------------------------------------------
+#
+# Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+# ----------------------------------------------------------------------------------------------------
+
+"""
+Simple timing wrapper that measures wall-clock time with high precision.
+Usage: python timing_wrapper.py <command> [args...]
+"""
+import sys
+import subprocess
+import time
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python timing_wrapper.py <command> [args...]", file=sys.stderr)
+        sys.exit(1)
+
+    cmd = sys.argv[1:]
+
+    start_time = time.perf_counter()
+
+    try:
+        result = subprocess.run(cmd, check=False)
+        elapsed = time.perf_counter() - start_time
+
+        print(f"Wall-clock time: {elapsed:.6f} sec", file=sys.stderr)
+        sys.exit(result.returncode)
+
+    except KeyboardInterrupt:
+        elapsed = time.perf_counter() - start_time
+        print(f"\nWall-clock time (interrupted): {elapsed:.6f} sec", file=sys.stderr)
+        sys.exit(1)
+
+    except OSError as e:
+        elapsed = time.perf_counter() - start_time
+        print(f"Error running command: {e}", file=sys.stderr)
+        print(f"Wall-clock time (error): {elapsed:.6f} sec", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

src/mx/mx_version.py

@@ -1,2 +1,2 @@
 # The version must be updated for every PR (checked in CI) and the comment should reflect the PR's issue
-version = "7.68.4"  # GR-71712 SpotBugs suite-level attribute must not override project level config
+version = "7.68.5"  # GR-70638 - benchmarking time tracker