update recommended sdk version to: 101370926 (#65)

Co-authored-by: Han Wang <[email protected]>

Author: wangchao1230

examples/flows/evaluation/basic-eval/README.md

@@ -19,10 +19,10 @@ Testing flow/node:
 pf flow test --flow .
 
 # test with flow inputs
-pf flow test --flow . --inputs groundtruth=groundtruth prediction=prediction
+pf flow test --flow . --inputs groundtruth=ABC prediction=ABC
 
 # test node with inputs
-pf flow test --flow . --node line_process --inputs groundtruth=groundtruth prediction=prediction
+pf flow test --flow . --node line_process --inputs groundtruth=ABC prediction=ABC
 ```
 
 ### 2. create flow run with multi line data

examples/flows/evaluation/basic-eval/aggregate.py

@@ -4,14 +4,11 @@
 
 
 @tool
-def aggregate(processed_results: List[str], variant_ids: List[str], line_numbers: List[int]):
+def aggregate(processed_results: List[str]):
     """
     This tool aggregates the processed result of all lines to the variant level and log metric for each variant.
 
     :param processed_results: List of the output of line_process node.
-    :param variant_ids: List of variant ids that can be used to group the results by variant.
-    :param line_numbers: List of line numbers of the variants. If provided, this can be used to
-                        group the results by line number.
     """
 
     # Add your aggregation logic here
@@ -28,6 +25,7 @@ def aggregate(processed_results: List[str], variant_ids: List[str], line_numbers
     #    },
     #    ...
     # }
+    print(len(processed_results))
     aggregated_results = {}
 
     # Log metric for each variant

examples/flows/evaluation/basic-eval/data.jsonl

@@ -1 +1 @@
-{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow.","variant_id": "variant_0","line_number": 0}
+{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow."}

examples/flows/evaluation/basic-eval/flow.dag.yaml

@@ -5,12 +5,6 @@ inputs:
   prediction:
     type: string
     default: prediction
-  variant_id:
-    type: string
-    default: variant_0
-  line_number:
-    type: int
-    default: 0
 outputs:
   results:
     type: string
@@ -31,8 +25,4 @@ nodes:
     path: aggregate.py
   inputs:
     processed_results: ${line_process.output}
-    variant_ids: ${inputs.variant_id}
-    line_numbers: ${inputs.line_number}
   aggregation: true
-id: template_eval_flow
-name: Template Evaluation Flow

examples/flows/evaluation/basic-eval/line_process.py

@@ -10,8 +10,5 @@ def line_process(groundtruth: str, prediction: str):
     :param prediction: the prediction of a single line.
     """
 
-    processed_result = ""
-
     # Add your line processing logic here
-
-    return processed_result
+    return "Correct" if groundtruth.lower() == prediction.lower() else "Incorrect"

examples/flows/evaluation/classification-accuracy-eval/README.md

@@ -19,7 +19,7 @@ In this flow, you will learn
 pf flow test --flow .
 
 # test with flow inputs
-pf flow test --flow . --inputs line_number=0 variant_id=variant_0
+pf flow test --flow . --inputs groundtruth=APP prediction=APP
 
 # test node with inputs
 pf flow test --flow . --node grade --inputs groundtruth=groundtruth prediction=prediction

examples/flows/evaluation/classification-accuracy-eval/calculate_accuracy.py

@@ -4,18 +4,14 @@
 
 
 @tool
-def calculate_accuracy(grades: List[str], variant_ids: List[str]):
-    aggregate_grades = {}
+def calculate_accuracy(grades: List[str]):
+    result = []
     for index in range(len(grades)):
         grade = grades[index]
-        variant_id = variant_ids[index]
-        if variant_id not in aggregate_grades.keys():
-            aggregate_grades[variant_id] = []
-        aggregate_grades[variant_id].append(grade)
+        result.append(grade)
 
     # calculate accuracy for each variant
-    for name, values in aggregate_grades.items():
-        accuracy = round((values.count("Correct") / len(values)), 2)
-        log_metric("accuracy", accuracy, variant_id=name)
+    accuracy = round((result.count("Correct") / len(result)), 2)
+    log_metric("accuracy", accuracy)
 
-    return aggregate_grades
+    return result

examples/flows/evaluation/classification-accuracy-eval/data.jsonl

@@ -1,3 +1,3 @@
-{"line_number": 0, "variant_id": "variant_0", "groundtruth": "App","prediction": "App"}
-{"line_number": 1, "variant_id": "variant_0", "groundtruth": "Channel","prediction": "Channel"}
-{"line_number": 2, "variant_id": "variant_0", "groundtruth": "Academic","prediction": "Academic"}
+{"groundtruth": "App","prediction": "App"}
+{"groundtruth": "Channel","prediction": "Channel"}
+{"groundtruth": "Academic","prediction": "Academic"}

examples/flows/evaluation/classification-accuracy-eval/flow.dag.yaml

@@ -1,10 +1,4 @@
 inputs:
-  line_number:
-    type: int
-    default: 0
-  variant_id:
-    type: string
-    default: variant_0
   groundtruth:
     type: string
     description: Please specify the groundtruth column, which contains the true label
@@ -35,5 +29,4 @@ nodes:
     path: calculate_accuracy.py
   inputs:
     grades: ${grade.output}
-    variant_ids: ${inputs.variant_id}
   aggregation: true

examples/requirements.txt

@@ -1,6 +1,6 @@
 # remove when we publish to pypi
 --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/
-promptflow[azure]==0.0.101192967
+promptflow[azure]==0.0.101370926
 promptflow-tools==0.0.1.dev0
 python-dotenv
 langchain