add generate metrics, and capitalize node names and metric names

Author: menishmueli

spark-ui/src/components/SqlFlow/StageNode.tsx

@@ -13,6 +13,7 @@ import { getSizeFromMetrics } from '../../reducers/PlanGraphUtils';
 import { truncateMiddle } from "../../reducers/PlanParsers/PlanParserUtils";
 import {
   calculatePercentage,
+  capitalizeWords,
   humanFileSize,
   humanizeTimeDiff,
   parseBytesString,
@@ -678,6 +679,8 @@ export const StageNode: FC<{
     });
   }
 
+  const dataTableWithCapitalizedNames = dataTable.map(metric => ({ ...metric, name: capitalizeWords(metric.name) }))
+
   return (
     <>
       <Handle type="target" position={Position.Left} id="b" />
@@ -695,7 +698,7 @@ export const StageNode: FC<{
             >
               {data.node.enrichedName}
             </Typography>
-            {dataTable.map((metric) => (
+            {dataTableWithCapitalizedNames.map((metric) => (
               <ConditionalWrapper
                 key={metric.name}
                 condition={metric.tooltip !== undefined}

spark-ui/src/reducers/SqlReducer.ts

@@ -18,6 +18,7 @@ import { SparkSQL, SparkSQLs, SqlStatus } from "../interfaces/SparkSQLs";
 import { NodesMetrics } from "../interfaces/SqlMetrics";
 import {
   calculatePercentage,
+  capitalizeWords,
   timeStrToEpocTime,
   timeStringToMilliseconds,
 } from "../utils/FormatUtils";
@@ -222,7 +223,7 @@ function calculateSql(
       rddScopeId: nodePlan?.rddScopeId,
       type: type,
       parsedPlan: parsedPlan,
-      enrichedName: nodeEnrichedNameBuilder(node.nodeName, parsedPlan),
+      enrichedName: capitalizeWords(nodeEnrichedNameBuilder(node.nodeName, parsedPlan)),
       isCodegenNode: isCodegenNode,
       wholeStageCodegenId: isCodegenNode
         ? extractCodegenId()
@@ -702,6 +703,36 @@ function updateParsedPlan(
   return node.parsedPlan;
 }
 
+function addGenerateMetrics(
+  node: EnrichedSqlNode,
+  updatedMetrics: EnrichedSqlMetric[],
+  graph: Graph,
+  allNodes: EnrichedSqlNode[],
+): EnrichedSqlMetric | null {
+  if (node.nodeName === "Generate") {
+    const inputNode = findLastNodeWithInputRows(node, graph, allNodes);
+    if (!inputNode) {
+      return null;
+    }
+
+    const inputRows = getRowsFromMetrics(inputNode.metrics);
+    if (inputRows === null || inputRows === 0) {
+      return null;
+    }
+
+    const outputRows = getRowsFromMetrics(updatedMetrics);
+    if (outputRows === null) {
+      return null;
+    }
+
+    const ratio = outputRows / inputRows;
+    const ratioFormatted = ratio.toFixed(2);
+
+    return { name: `${node.enrichedName} Ratio`, value: `${ratioFormatted}X` };
+  }
+  return null;
+}
+
 function updateNodeMetrics(
   node: EnrichedSqlNode,
   metrics: EnrichedSqlMetric[],
@@ -712,6 +743,7 @@ function updateNodeMetrics(
   const filterRatio = addFilterRatioMetric(node, updatedOriginalMetrics, graph, allNodes);
   const crossJoinFilterRatio = addCrossJoinFilterRatioMetric(node, updatedOriginalMetrics, graph, allNodes);
   const joinMetrics = addJoinMetrics(node, updatedOriginalMetrics, graph, allNodes);
+  const generateMetrics = addGenerateMetrics(node, updatedOriginalMetrics, graph, allNodes);
   return [
     ...updatedOriginalMetrics,
     ...(filterRatio !== null
@@ -723,6 +755,9 @@ function updateNodeMetrics(
     ...(joinMetrics !== null
       ? joinMetrics
       : []),
+    ...(generateMetrics !== null
+      ? [generateMetrics]
+      : []),
   ];
 }
 

spark-ui/src/utils/FormatUtils.ts

@@ -11,6 +11,13 @@ export function humanFileSize(bytes: number): string {
     .replace("TB", "TiB");
 }
 
+export function capitalizeWords(text: string): string {
+  return text
+    .split(' ')
+    .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
+    .join(' ');
+}
+
 export function parseBytesString(str: string): number {
   return parse(
     str