Samples for VLM video input.

samples/cpp/visual_language_chat/CMakeLists.txt

@@ -55,3 +55,24 @@ install(TARGETS benchmark_vlm
         RUNTIME DESTINATION samples_bin/
         COMPONENT samples_bin
         EXCLUDE_FROM_ALL)
+
+
+include(FetchContent)
+FetchContent_Declare(
+        opencv
+        GIT_REPOSITORY https://github.com/opencv/opencv.git
+        GIT_TAG 4.12.0
+        GIT_SHALLOW TRUE
+        GIT_PROGRESS TRUE
+)
+FetchContent_MakeAvailable(opencv)
+
+
+add_executable(video_to_text_chat video_to_text_chat.cpp)
+
+target_include_directories(video_to_text_chat PRIVATE
+        ${OPENCV_CONFIG_FILE_INCLUDE_DIR}
+        ${OPENCV_MODULE_opencv_core_LOCATION}/include
+        ${OPENCV_MODULE_opencv_videoio_LOCATION}/include
+        )
+target_link_libraries(video_to_text_chat opencv_core opencv_videoio openvino::genai cxxopts::cxxopts)

samples/cpp/visual_language_chat/README.md

@@ -3,8 +3,9 @@
 This example showcases inference of Visual language models (VLMs). The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample features `ov::genai::VLMPipeline` and runs the simplest deterministic greedy sampling algorithm. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/latest/notebooks/minicpm-v-multimodal-chatbot) which provides an example of Visual-language assistant.
 
 
-There are two sample files:
+There are three sample files:
  - [`visual_language_chat.cpp`](./visual_language_chat.cpp) demonstrates basic usage of the VLM pipeline.
+ - [`video_to_text_chat.cpp`](./video_to_text_chat.cpp) demonstrates video to text usage of the VLM pipeline.
  - [`benchmark_vlm.cpp`](./benchmark_vlm.cpp) shows how to benchmark a VLM in OpenVINO GenAI. The script includes functionality for warm-up iterations, generating text and calculating various performance metrics.
 
 
@@ -19,9 +20,9 @@ pip install --upgrade-strategy eager -r ../../requirements.txt
 optimum-cli export openvino --model openbmb/MiniCPM-V-2_6 --trust-remote-code MiniCPM-V-2_6
 ```
 
-## Run
+Follow [Get Started with Samples](https://docs.openvino.ai/2025/get-started/learn-openvino/openvino-samples/get-started-demos.html) to run samples.
 
-Follow [Get Started with Samples](https://docs.openvino.ai/2025/get-started/learn-openvino/openvino-samples/get-started-demos.html) to run the sample.
+## Run visual language chat:
 
 [This image](https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11) can be used as a sample image.
 
@@ -31,6 +32,17 @@ Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is
 
 Refer to the [Supported Models](https://openvinotoolkit.github.io/openvino.genai/docs/supported-models/#visual-language-models-vlms) for more details.
 
+
+## Run video to text chat:
+
+To run this sample a model that supports video input is required, for example `llava-hf/LLaVA-NeXT-Video-7B-hf`.
+
+[This video](https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4) can be used as a sample video.
+
+`video_to_text_chat ./LLaVA-NeXT-Video-7B-hf/ sample_demo_1.mp4`
+
+Supported models with video input are listed in [this section](https://openvinotoolkit.github.io/openvino.genai/docs/use-cases/image-processing/#use-image-or-video-tags-in-prompt).
+
 ## Run benchmark:
 
 ```sh

samples/cpp/visual_language_chat/video_to_text_chat.cpp

@@ -0,0 +1,124 @@
+// Copyright (C) 2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include <openvino/genai/visual_language/pipeline.hpp>
+#include <opencv2/core.hpp>
+#include <opencv2/videoio.hpp>
+#include <iostream>
+#include <filesystem>
+
+namespace fs = std::filesystem;
+
+std::vector<size_t> make_indices(size_t total_frames, size_t num_frames) {
+    std::vector<size_t> indices;
+    indices.reserve(num_frames);
+
+    auto step = float(total_frames) / num_frames;
+
+    for (size_t i = 0; i < num_frames; ++i) {
+        size_t idx = std::min(size_t(i * step), total_frames - 1);
+        indices.push_back(idx);
+    }
+
+    return indices;
+}
+
+ov::Tensor load_video(const std::filesystem::path& video_path, size_t num_frames = 10) {
+    cv::VideoCapture cap(video_path.string());
+
+    if (!cap.isOpened()) {
+        OPENVINO_THROW("Could not open the video file.");
+    }
+    size_t total_num_frames = cap.get(cv::CAP_PROP_FRAME_COUNT);
+    auto indices = make_indices(total_num_frames, num_frames);
+
+    std::vector<cv::Mat> frames;
+    cv::Mat frame;
+    size_t width  = cap.get(cv::CAP_PROP_FRAME_WIDTH);
+    size_t height = cap.get(cv::CAP_PROP_FRAME_HEIGHT);
+    ov::Tensor video_tensor(ov::element::u8, ov::Shape{num_frames, height, width, 3});
+    auto video_tensor_data = video_tensor.data<uint8_t>();
+
+    size_t frame_idx = 0;
+    while (cap.read(frame)) {
+        if (std::find(indices.begin(), indices.end(), frame_idx) != indices.end()) {
+            memcpy(video_tensor_data, frame.data, frame.total() * 3 * sizeof(uint8_t));
+            video_tensor_data += frame.total() * 3;
+        }
+        frame_idx++;
+    }
+
+    return video_tensor;
+}
+
+std::vector<ov::Tensor> load_videos(const std::filesystem::path& input_path) {
+    if (input_path.empty() || !fs::exists(input_path)) {
+        throw std::runtime_error{"Path to images is empty or does not exist."};
+    }
+    if (fs::is_directory(input_path)) {
+        std::set<fs::path> sorted_videos{fs::directory_iterator(input_path), fs::directory_iterator()};
+        std::vector<ov::Tensor> videos;
+        for (const fs::path& dir_entry : sorted_videos) {
+            videos.push_back(load_video(dir_entry));
+        }
+        return videos;
+    }
+    return {load_video(input_path)};
+}
+
+bool print_subword(std::string&& subword) {
+    return !(std::cout << subword << std::flush);
+}
+
+int main(int argc, char* argv[]) try {
+    if (argc < 3 || argc > 4) {
+        throw std::runtime_error(std::string{"Usage "} + argv[0] + " <MODEL_DIR> <VIDEO_FILE OR DIR_WITH_VIDEOS> <DEVICE>");
+    }
+
+    std::vector<ov::Tensor> videos = load_videos(argv[2]);
+
+    // GPU and NPU can be used as well.
+    // Note: If NPU is selected, only language model will be run on NPU
+    std::string device = (argc == 4) ? argv[3] : "CPU";
+    ov::AnyMap enable_compile_cache;
+    if (device == "GPU") {
+        // Cache compiled models on disk for GPU to save time on the
+        // next run. It's not beneficial for CPU.
+        enable_compile_cache.insert({ov::cache_dir("vlm_cache")});
+    }
+    ov::genai::VLMPipeline pipe(argv[1], device, enable_compile_cache);
+
+    ov::genai::GenerationConfig generation_config;
+    generation_config.max_new_tokens = 100;
+
+    std::string prompt;
+
+    pipe.start_chat();
+    std::cout << "question:\n";
+
+    std::getline(std::cin, prompt);
+    pipe.generate(prompt,
+                  ov::genai::videos(videos),
+                  ov::genai::generation_config(generation_config),
+                  ov::genai::streamer(print_subword));
+    std::cout << "\n----------\n"
+        "question:\n";
+    while (std::getline(std::cin, prompt)) {
+        pipe.generate(prompt,
+                      ov::genai::generation_config(generation_config),
+                      ov::genai::streamer(print_subword));
+        std::cout << "\n----------\n"
+            "question:\n";
+    }
+    pipe.finish_chat();
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+}

samples/deployment-requirements.txt

@@ -4,3 +4,4 @@ librosa==0.11.0  # For Whisper
 pillow==12.0.0  # Image processing for VLMs
 json5==0.12.1  # For ReAct
 pydantic==2.12.4  # For Structured output json schema
+opencv-python # For video-to-text VLM sample

samples/python/visual_language_chat/README.md

@@ -2,8 +2,9 @@
 
 This example showcases inference of text-generation Vision Language Models (VLMs): `miniCPM-V-2_6` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample features `openvino_genai.VLMPipeline` and configures it for the chat scenario. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/latest/notebooks/minicpm-v-multimodal-chatbot) which provides an example of Visual-language assistant.
 
-There are two sample files:
+There are three sample files:
  - [`visual_language_chat.py`](./visual_language_chat.py) demonstrates basic usage of the VLM pipeline.
+ - [`video_to_text_chat.py`](./video_to_text_chat.py) demonstrates video to text usage of the VLM pipeline.
  - [`benchmark_vlm.py`](./benchmark_vlm.py) shows how to benchmark a VLM in OpenVINO GenAI. The script includes functionality for warm-up iterations, generating text and calculating various performance metrics.
 
 ## Download and convert the model and tokenizers
@@ -38,20 +39,29 @@ tokenizer = AutoTokenizer.from_pretrained("openbmb/MiniCPM-V-2_6")
 export_tokenizer(tokenizer, output_dir)
 ```
 
-## Run:
+Install [deployment-requirements.txt](../../deployment-requirements.txt) via `pip install -r ../../deployment-requirements.txt` to run VLM samples.
 
-[This image](https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11) can be used as a sample image.
+## Run image-to-text chat sample:
 
-Install [deployment-requirements.txt](../../deployment-requirements.txt) via `pip install -r ../../deployment-requirements.txt` and then, run a sample:
+[This image](https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11) can be used as a sample image.
 
 `python visual_language_chat.py ./miniCPM-V-2_6/ 319483352-d5fbbd1a-d484-415c-88cb-9986625b7b11.jpg`
 
+See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models.
+
+## Run video-to-text chat sample:
+
+To run this sample a model that supports video input is required, for example `llava-hf/LLaVA-NeXT-Video-7B-hf`.
+
+[This video](https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4) can be used as a sample video.
+
+`python video_to_text_chat.py ./LLaVA-NeXT-Video-7B-hf/ sample_demo_1.mp4`
+
+Supported models with video input are listed in [this section](https://openvinotoolkit.github.io/openvino.genai/docs/use-cases/image-processing/#use-image-or-video-tags-in-prompt).
 
 Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is recommended to run larger models on a dGPU with 32GB+ RAM. # TODO: examples of larger models
 Modify the source code to change the device for inference to the GPU.
 
-See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models.
-
 ## Run benchmark:
 
 ```sh

samples/python/visual_language_chat/video_to_text_chat.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+
+import argparse
+import numpy as np
+import cv2
+import openvino_genai
+from openvino import Tensor
+from pathlib import Path
+
+
+def streamer(subword: str) -> bool:
+    '''
+
+    Args:
+        subword: sub-word of the generated text.
+
+    Returns: Return flag corresponds whether generation should be stopped.
+
+    '''
+    print(subword, end='', flush=True)
+
+    # No value is returned as in this example we don't want to stop the generation in this method.
+    # "return None" will be treated the same as "return openvino_genai.StreamingStatus.RUNNING".
+
+
+def read_video(path: str, num_frames: int = 10) -> Tensor:
+    '''
+
+    Args:
+        path: The path to the video.
+
+    Returns: the ov.Tensor containing the video.
+
+    '''
+    cap = cv2.VideoCapture(path)
+
+    frames = []
+
+    while cap.isOpened():
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frames.append(np.array(frame))
+
+    indices = np.arange(0, len(frames), len(frames) / num_frames).astype(int)
+    frames = [frames[i] for i in indices]
+
+    return Tensor(frames)
+
+
+def read_videos(path: str) -> list[Tensor]:
+    entry = Path(path)
+    if entry.is_dir():
+        return [read_video(str(file)) for file in sorted(entry.iterdir())]
+    return [read_video(path)]
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('model_dir', help="Path to the model directory")
+    parser.add_argument('video_dir', help="Path to a video file.")
+    parser.add_argument('device', nargs='?', default='CPU', help="Device to run the model on (default: CPU)")
+    args = parser.parse_args()
+
+    videos = read_videos(args.video_dir)
+
+    # GPU and NPU can be used as well.
+    # Note: If NPU is selected, only the language model will be run on the NPU.
+    enable_compile_cache = dict()
+    if args.device == "GPU":
+        # Cache compiled models on disk for GPU to save time on the next run.
+        # It's not beneficial for CPU.
+        enable_compile_cache["CACHE_DIR"] = "vlm_cache"
+
+    pipe = openvino_genai.VLMPipeline(args.model_dir, args.device, **enable_compile_cache)
+
+    config = openvino_genai.GenerationConfig()
+    config.max_new_tokens = 100
+
+    pipe.start_chat()
+    prompt = input('question:\n')
+    pipe.generate(prompt, videos=videos, generation_config=config, streamer=streamer)
+
+    while True:
+        try:
+            prompt = input("\n----------\n"
+                "question:\n")
+        except EOFError:
+            break
+        pipe.generate(prompt, generation_config=config, streamer=streamer)
+    pipe.finish_chat()
+
+
+if __name__ == '__main__':
+    main()

tests/python_tests/samples/conftest.py

@@ -143,6 +143,10 @@
     "tiny-random-SpeechT5ForTextToSpeech": {
         "name": "hf-internal-testing/tiny-random-SpeechT5ForTextToSpeech",
         "convert_args": ["--model-kwargs",  json.dumps({"vocoder": "fxmarty/speecht5-hifigan-tiny"})]
+    },
+    "tiny-random-llava-next-video": {
+        "name": "optimum-intel-internal-testing/tiny-random-llava-next-video",
+        "convert_args": ["--task", "image-text-to-text"]
     }
 }
 
@@ -159,7 +163,8 @@
     "cat.png": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
     "cat": "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11",
     "3283_1447_000.tar.gz": "https://huggingface.co/datasets/facebook/multilingual_librispeech/resolve/main/data/mls_polish/train/audio/3283_1447_000.tar.gz",
-    "cmu_us_awb_arctic-wav-arctic_a0001.bin": "https://huggingface.co/datasets/Xenova/cmu-arctic-xvectors-extracted/resolve/main/cmu_us_awb_arctic-wav-arctic_a0001.bin"
+    "cmu_us_awb_arctic-wav-arctic_a0001.bin": "https://huggingface.co/datasets/Xenova/cmu-arctic-xvectors-extracted/resolve/main/cmu_us_awb_arctic-wav-arctic_a0001.bin",
+    "videos/sample_video.mp4": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4",
 }
 
 SAMPLES_PY_DIR = Path(os.environ.get("SAMPLES_PY_DIR", os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../samples/python"))))

tests/python_tests/samples/test_video_to_text_chat.py

@@ -0,0 +1,34 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+import pytest
+import subprocess # nosec B404
+import sys
+
+from conftest import SAMPLES_PY_DIR, SAMPLES_CPP_DIR, SAMPLES_C_DIR
+from test_utils import run_sample
+
+class TestVisualLanguageChat:
+    @pytest.mark.vlm
+    @pytest.mark.samples
+    @pytest.mark.parametrize(
+        "convert_model, download_test_content, questions",
+        [
+            pytest.param("tiny-random-llava-next-video", "videos/sample_video.mp4", 'What is unusual on this video?\nGo on.')
+        ],
+        indirect=["convert_model", "download_test_content"],
+    )
+    def test_sample_visual_language_chat(self, convert_model, download_test_content, questions):
+        # Test CPP sample
+        cpp_sample = os.path.join(SAMPLES_CPP_DIR, 'video_to_text_chat')
+        cpp_command = [cpp_sample, convert_model, download_test_content]
+        cpp_result = run_sample(cpp_command, questions)
+
+        # Test Python sample
+        py_script = os.path.join(SAMPLES_PY_DIR, "visual_language_chat/video_to_text_chat.py")
+        py_command = [sys.executable, py_script, convert_model, download_test_content]
+        py_result = run_sample(py_command, questions)
+
+        # Compare results
+        assert py_result.stdout == cpp_result.stdout, f"Results should match"