Merge branch 'main' into release/promptflow/1.15.0.post1

Author: singankit

.cspell.json

@@ -54,65 +54,67 @@
     "benchmark/promptflow-serve/result-archive/**"
   ],
   "words": [
-    "aoai",
     "amlignore",
-    "mldesigner",
-    "faiss",
-    "serp",
+    "aoai",
+    "Apim",
+    "astext",
+    "attribited",
+    "azureai",
+    "azurecr",
     "azureml",
-    "mlflow",
-    "vnet",
-    "openai",
-    "pfazure",
+    "azuremlsdktestpypi",
+    "Bhavik",
+    "centralus",
+    "chatml",
+    "cref",
+    "e2etest",
+    "e2etests",
     "eastus",
-    "azureai",
-    "vectordb",
-    "Qdrant",
-    "Weaviate",
+    "Entra",
     "env",
-    "e2etests",
-    "e2etest",
-    "tablefmt",
-    "logprobs",
-    "logit",
+    "faiss",
+    "geval",
     "hnsw",
-    "chatml",
-    "UNLCK",
+    "junit",
     "KHTML",
+    "Likert",
+    "llmlingua",
+    "logit",
+    "logprobs",
+    "meid",
+    "mgmt",
+    "MistralAI",
+    "mldesigner",
+    "mlflow",
+    "msal",
+    "msrest",
+    "myconn",
     "numlines",
-    "azurecr",
-    "centralus",
+    "nunit",
+    "openai",
+    "pfazure",
+    "pfbytes",
+    "pfcli",
+    "pfutil",
     "Policheck",
-    "azuremlsdktestpypi",
-    "rediraffe",
     "pydata",
-    "ROBOCOPY",
-    "undoc",
+    "Qdrant",
+    "rediraffe",
     "retriable",
-    "pfcli",
-    "pfutil",
-    "mgmt",
-    "wsid",
-    "westus",
-    "msrest",
-    "cref",
-    "msal",
-    "pfbytes",
-    "Apim",
-    "junit",
-    "nunit",
-    "astext",
-    "Likert",
-    "geval",
+    "ROBOCOPY",
+    "serp",
     "Summ",
-    "Bhavik",
-    "meid",
-    "Entra",
+    "tablefmt",
+    "undoc",
+    "UNLCK",
+    "upia",
     "uvicorn",
-    "attribited",
-    "MistralAI",
-    "llmlingua",
-    "myconn"
+    "vectordb",
+    "vnet",
+    "Weaviate",
+    "westus",
+    "wsid",
+    "Xpia"
   ],
   "ignoreWords": [
     "openmpi",
@@ -242,10 +244,20 @@
     "azureopenaimodelconfiguration",
     "openaimodelconfiguration",
     "usecwd",
+    "upia",
+    "xpia",
     "locustio",
     "euap",
+    "Rerank",
+    "rerank",
+    "reranker",
     "rcfile",
-    "pylintrc"
+    "pylintrc",
+    "gleu",
+    "Gleu",
+    "GLEU",
+    "fmeasure",
+    "punkt"
   ],
   "flagWords": [
     "Prompt Flow"

docs/cloud/index.md

@@ -27,12 +27,6 @@ In prompt flow, You can develop your flow locally and then seamlessly transition
 
 For more resources on Azure AI, visit the cloud documentation site: [Build AI solutions with prompt flow](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/get-started-prompt-flow?view=azureml-api-2).
 
-```{toctree}
-:caption: Tracing
-:maxdepth: 2
-azureai/tracing/index
-```
-
 ```{toctree}
 :caption: Flow
 :maxdepth: 2

docs/concepts/concept-connections.md

@@ -11,11 +11,11 @@ Prompt flow provides a variety of pre-built connections, including Azure OpenAI,
 
 | Connection type                                              | Built-in tools                  |
 | ------------------------------------------------------------ | ------------------------------- |
-| [Azure OpenAI](https://azure.microsoft.com/en-us/products/cognitive-services/openai-service) | LLM or Python                   |
+| [Azure OpenAI](https://azure.microsoft.com/products/cognitive-services/openai-service) | LLM or Python                   |
 | [OpenAI](https://openai.com/)                               | LLM or Python                   |
-| [Cognitive Search](https://azure.microsoft.com/en-us/products/search) | Vector DB Lookup or Python      |
+| [Cognitive Search](https://azure.microsoft.com/products/search) | Vector DB Lookup or Python      |
 | [Serp](https://serpapi.com/)                                 | Serp API or Python              |
-| [Serverless](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#deploy-models-with-model-as-a-service-maas)                                               | LLM or Python                   |
+| [Serverless](https://learn.microsoft.com/azure/ai-studio/concepts/deployments-overview#deploy-models-as-serverless-apis)                                               | LLM or Python                   |
 | Custom                                                       | Python                          |
 
 By leveraging connections in prompt flow, you can easily establish and manage connections to external APIs and data sources, facilitating efficient data exchange and interaction within their AI applications.

docs/reference/index.md

@@ -66,6 +66,7 @@ tools-reference/open_model_llm_tool
 tools-reference/openai-gpt-4v-tool
 tools-reference/contentsafety_text_tool
 tools-reference/aoai-gpt4-turbo-vision
+tools-reference/rerank-tool
 ```
 
 ```{toctree}

docs/reference/pf-command-reference.md

@@ -1,9 +1,5 @@
 # pf
 
-:::{admonition} Experimental feature
-This is an experimental feature, and may change at any time. Learn [more](../how-to-guides/faq.md#stable-vs-experimental).
-:::
-
 Manage prompt flow resources with the prompt flow CLI.
 
 | Command                         | Description                    |

docs/reference/tools-reference/llm-tool.md

@@ -25,7 +25,7 @@ Create OpenAI resources, Azure OpenAI resources or MaaS deployment with the LLM
 
 - **MaaS deployment**
 
-    Create MaaS deployment for models in Azure AI Studio model catalog with [instruction](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/deployments-overview#deploy-models-with-model-as-a-service-maas)
+    Create MaaS deployment for models in Azure AI Studio model catalog with [instruction](https://learn.microsoft.com/azure/ai-studio/concepts/deployments-overview#deploy-models-as-serverless-apis)
 
     You can create serverless connection to use this MaaS deployment.
 

docs/reference/tools-reference/rerank-tool.md

@@ -0,0 +1,69 @@
+# Rerank
+
+## Introduction
+Rerank is a semantic search tool that improves search quality with a semantic-based reranking system which can contextualize the meaning of a user's query beyond keyword relevance. This tool works best with look up tool as a ranker after the initial retrieval. The list of current supported ranking method is as follows.
+
+| Name | Description |
+| --- | --- |
+| BM25 | BM25 is an open source ranking algorithm to measure the relevance of documents to a given query |
+| Scaled Score Fusion | Scaled Score Fusion calculates a scaled relevance score. |
+| Cohere Rerank | Cohere Rerank is the market’s leading reranking model used for semantic search and retrieval-augmented generation (RAG). |
+
+## Requirements
+- For AzureML users, the tool is installed in default image, you can use the tool without extra installation.
+- For local users,
+
+  `pip install promptflow-vectordb`
+
+## Prerequisites
+
+BM25 and Scaled Score Fusion are included as default reranking methods. To use cohere rerank model, you should create serverless deployment to the model, and establish connection between the tool and the resource as follows.
+
+- Add Serverless Model connection. Fill "API base" and "API key" field to your serverless deployment.
+
+
+## Inputs
+
+|  Name                  | Type        | Description                                                           | Required |
+|------------------------|-------------|-----------------------------------------------------------------------|----------|
+| query                  | string      | the question relevant to your input documents                         | Yes      |
+| ranker_parameters      | string      | the type of ranking methods to use                                    | Yes      |
+| result_groups          | object      | the list of document chunks to rerank. Normally this is output from lookup | Yes      |
+| top_k                  | int         | the maximum number of relevant documents to return                    | No      |
+
+
+
+## Outputs
+
+| Return Type | Description                              |
+|-------------|------------------------------------------|
+| text        | text of the entity |
+| metadata    | metadata like file path and url |
+| additional_fields    | metadata and rerank score |
+
+  <details>
+    <summary>Output</summary>
+    
+  ```json
+  [
+    {
+        "text": "sample text",
+        "metadata":
+        {
+            "filepath": "sample_file_path",
+            "metadata_json_string": "meta_json_string"
+            "title": "",
+            "url": ""
+        },
+        "additional_fields":
+        {
+            "filepath": "sample_file_path",
+            "metadata_json_string": "meta_json_string"
+            "title": "",
+            "url": "",
+            "@promptflow_vectordb.reranker_score": 0.013795365
+        }
+    }
+  ]
+  ```
+  </details>

examples/flows/integrations/continuous-monitoring-with-pipeline/README.md

@@ -0,0 +1,69 @@
+# Continuous Monitoring Pipeline
+
+This tutorial describes an advanced use case of [running flows in Azure ML Pipelines](https://github.com/microsoft/promptflow/blob/main/examples/tutorials/run-flow-with-pipeline/pipeline.ipynb).  
+The detailed explanations of the prerequisites and principles can be found in the aforementioned article.  
+Continuous monitoring is necessary to maintain the quality, performance and efficiency of Generative AI applications.  
+These factors directly impact the user experience and operational costs.  
+
+We will run evaluations on a basic chatbot flow, then aggregate the results to export and visualize the metrics.  
+The flows used in this pipeline are described below:
+- [Basic Chat](https://github.com/microsoft/promptflow/tree/main/examples/flows/chat/chat-basic)
+- [Q&A Evaluation](https://github.com/microsoft/promptflow/tree/main/examples/flows/evaluation/eval-qna-rag-metrics)
+- [Perceived Intelligence Evaluation](https://github.com/microsoft/promptflow/tree/main/examples/flows/evaluation/eval-perceived-intelligence)
+- [Summarization Evaluation](https://github.com/microsoft/promptflow/tree/main/examples/flows/evaluation/eval-summarization)
+
+Connections used in this flow:
+- `azure_open_ai_connection` connection (Azure OpenAI).
+
+## Prerequisites
+
+### Prompt flow SDK:
+- Azure cloud setup:
+  - An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)
+  - Create an Azure ML resource from Azure portal - [Create a Azure ML workspace](https://ms.portal.azure.com/#view/Microsoft_Azure_Marketplace/MarketplaceOffersBlade/searchQuery/machine%20learning)
+  - Connect to your workspace then setup a basic computer cluster - [Configure workspace](https://github.com/microsoft/promptflow/blob/main/examples/configuration.ipynb)
+- Local environment setup:
+  - A python environment
+  - Installed Azure Machine Learning Python SDK v2 - [install instructions](https://github.com/microsoft/promptflow/blob/main/examples/README.md) - check the getting started section and make sure version of 'azure-ai-ml' is higher than `1.12.0`
+
+Note: when using the Prompt flow SDK, it may be useful to also install the [`Prompt flow for VS Code`](https://marketplace.visualstudio.com/items?itemName=prompt-flow.prompt-flow) extension (if using VS Code).
+
+### Azure AI/ML Studio:
+Start a compute session.  
+The follow the installation steps described in the notebook.
+
+## Setup connections
+Ensure that you have a connection to Azure OpenAI with the following deployments:
+- `gpt-35-turbo`
+- `gpt-4`
+
+## Run pipeline
+
+Run the notebook's steps until `3.2.2 Submit the job` to start the pipeline in Azure ML Studio.
+
+## Pipeline description
+The first node reads the evaluation dataset.  
+The second node is the main flow that will be monitored, it takes the output of the evaluation dataset as a `data` input.  
+After the main flow's node has completed, its output will go to 3 nodes:
+- Q&A Evaluation
+- Perceived Intelligence Evaluation
+- Simple Summarization
+
+The Simple Summarization and the main nodes' outputs will become the Summarization Evaluation node's input.
+
+Finally, all the evaluation metrics will be aggregated and displayed in Azure ML Pipeline's interface.
+
+![continuous_monitoring_pipeline.png](./monitoring/media/continuous_monitoring_pipeline.png)
+
+## Metrics visualization
+In the node `Convert evaluation results to parquet` Metrics tab, the aggregated metrics will be displayed.
+
+![metrics_tab.png](./monitoring/media/metrics_tab.png)
+
+The evolution of the metrics can be monitored by comparing multiple pipeline runs:  
+
+![compare_button.png](./monitoring/media/compare_button.png)
+
+![compare_metrics.png](./monitoring/media/compare_metrics.png)
+## Contact
+Please reach out to Lou Bigard (<[email protected]>) with any issues.

examples/flows/integrations/continuous-monitoring-with-pipeline/flows/standard/simple-summarization/flow.dag.yaml

@@ -0,0 +1,45 @@
+$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Flow.schema.json
+environment:
+  python_requirements_txt: requirements.txt
+inputs:
+  answer:
+    type: string
+outputs:
+  summary:
+    type: string
+    reference: ${summarize_text_content.output}
+nodes:
+- name: summarize_text_content
+  use_variants: true
+node_variants:
+  summarize_text_content:
+    default_variant_id: variant_0
+    variants:
+      variant_0:
+        node:
+          type: llm
+          source:
+            type: code
+            path: summarize_text_content.jinja2
+          inputs:
+            deployment_name: gpt-35-turbo
+            model: gpt-3.5-turbo
+            max_tokens: 128
+            temperature: 0.2
+            text: ${inputs.answer}
+          connection: open_ai_connection
+          api: chat
+      variant_1:
+        node:
+          type: llm
+          source:
+            type: code
+            path: summarize_text_content__variant_1.jinja2
+          inputs:
+            deployment_name: gpt-35-turbo
+            model: gpt-3.5-turbo
+            max_tokens: 256
+            temperature: 0.3
+            text: ${inputs.answer}
+          connection: open_ai_connection
+          api: chat

examples/flows/integrations/continuous-monitoring-with-pipeline/flows/standard/simple-summarization/requirements.txt

@@ -0,0 +1,2 @@
+promptflow[azure]>=1.7.0
+promptflow-tools