From d5b878717ff1112cf5f249236ff0aca0f3e8b54c Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Thu, 9 Jan 2025 11:21:37 +0100 Subject: [PATCH 1/2] Fix device format for OpenVINO Runtime --- .../evaluators/custom_evaluators/whisper_evaluator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py b/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py index 01a8fd81a6..9cd82a6b93 100644 --- a/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py +++ b/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py @@ -1,5 +1,5 @@ """ -Copyright (c) 2024 Intel Corporation +Copyright (c) 2024-2025 Intel Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -54,7 +54,7 @@ def from_configs(cls, config, delayed_model_loading=False, orig_config=None): dataset_config = config["datasets"] pipeline_class_name = config["pipeline_class"] if 'device' in config['launchers'][0]: - config["_device"] = config['launchers'][0]['device'] + config["_device"] = config['launchers'][0]['device'].upper() if pipeline_class_name not in cls.VALID_PIPELINE_CLASSES: raise ValueError(f"Invalid pipeline class name: {pipeline_class_name}. " From f5c1101dc6875a6936f43e26ca8f5434430b6dfa Mon Sep 17 00:00:00 2001 From: Piotr Wolnowski Date: Thu, 9 Jan 2025 12:38:30 +0100 Subject: [PATCH 2/2] Update README.md with whisper evaluator --- .../evaluators/custom_evaluators/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/README.md b/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/README.md index 877236a70c..f8af4c6db5 100644 --- a/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/README.md +++ b/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/README.md @@ -54,10 +54,15 @@ Optionally you can provide `module_config` section which contains config for cus Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/sr_evaluator.py`. * **Tacotron2 Evaluator** demonstrates how to evaluate custom Tacotron2 model for text to speech task. - Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/tacotron2_evaluator.py` + Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/tacotron2_evaluator.py`. * **Open-NMT Evaluator** demonstrates how to evaluate Open-NMT model for text translation task. - Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/opennmt_encoder_decoder_generator_evaluator.py` + Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/opennmt_encoder_decoder_generator_evaluator.py`. * **StableDiffusion Evaluator** demonstrates how to evaluate image generation for Stable Diffusion models family. - Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/stable_diffusion_evaluator.py` + Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/stable_diffusion_evaluator.py`. + +* **Whisper Evaluator** demonstrates how to evaluate Whisper family models with various pipeline classes, including GenAIWhisperPipeline, HFWhisperPipeline, and OptimumWhisperPipeline. + Evaluator code: `/tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/whisper_evaluator.py`. + +