Introduce AutoPipelineForText2Video (simple)

Author: naomili0924

auto_pipeline_test.py

@@ -0,0 +1,8 @@
+import torch
+from diffusers import AutoPipelineForText2Video
+from diffusers.utils import export_to_video
+
+pipe = AutoPipelineForText2Video.from_pretrained(
+    "THUDM/CogVideoX-5b",
+    torch_dtype=torch.bfloat16,
+)

src/diffusers/__init__.py

@@ -303,6 +303,7 @@
             "AutoPipelineForImage2Image",
             "AutoPipelineForInpainting",
             "AutoPipelineForText2Image",
+            "AutoPipelineForText2Video",
             "ConsistencyModelPipeline",
             "DanceDiffusionPipeline",
             "DDIMPipeline",

src/diffusers/pipelines/__init__.py

@@ -46,6 +46,7 @@
         "AutoPipelineForImage2Image",
         "AutoPipelineForInpainting",
         "AutoPipelineForText2Image",
+        "AutoPipelineForText2Video",
     ]
     _import_structure["consistency_models"] = ["ConsistencyModelPipeline"]
     _import_structure["dance_diffusion"] = ["DanceDiffusionPipeline"]

src/diffusers/pipelines/auto_pipeline.py

@@ -118,6 +118,8 @@
     StableDiffusionXLPipeline,
 )
 from .wan import WanImageToVideoPipeline, WanPipeline, WanVideoToVideoPipeline
+from .hunyuan_video import HunyuanVideoPipeline
+from .cogvideo import CogVideoXPipeline
 from .wuerstchen import WuerstchenCombinedPipeline, WuerstchenDecoderPipeline
 
 
@@ -218,6 +220,8 @@
 AUTO_TEXT2VIDEO_PIPELINES_MAPPING = OrderedDict(
     [
         ("wan", WanPipeline),
+        ("hunyuan", HunyuanVideoPipeline),
+        ("cogvideox", CogVideoXPipeline),
     ]
 )
 
@@ -1203,3 +1207,39 @@ def from_pipe(cls, pipeline, **kwargs):
         model.register_to_config(**unused_original_config)
 
         return model
+
+class AutoPipelineForText2Video(ConfigMixin):
+    
+    config_name = "model_index.json"
+
+    def __init__(self, *args, **kwargs):
+        raise EnvironmentError(
+            f"{self.__class__.__name__} is designed to be instantiated "
+            f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or "
+            f"`{self.__class__.__name__}.from_pipe(pipeline)` methods."
+        )
+
+    @classmethod
+    @validate_hf_hub_args
+    def from_pretrained(cls, pretrained_model_or_path, **kwargs):
+        cache_dir = kwargs.pop("cache_dir", None)
+        force_download = kwargs.pop("force_download", False)
+        proxies = kwargs.pop("proxies", None)
+        token = kwargs.pop("token", None)
+        local_files_only = kwargs.pop("local_files_only", False)
+        revision = kwargs.pop("revision", None)
+ 
+        load_config_kwargs = {
+            "cache_dir": cache_dir,
+            "force_download": force_download,
+            "proxies": proxies,
+            "token": token,
+            "local_files_only": local_files_only,
+            "revision": revision,
+        }
+
+        config = cls.load_config(pretrained_model_or_path, **load_config_kwargs)
+        orig_class_name = config["_class_name"]
+        text_to_video_cls = _get_task_class(AUTO_TEXT2VIDEO_PIPELINES_MAPPING, orig_class_name)
+        kwargs = {**load_config_kwargs, **kwargs}
+        return text_to_video_cls.from_pretrained(pretrained_model_or_path, **kwargs)

src/diffusers/pipelines/test.ipynb

@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "168c7d5f-bdb8-48e3-b696-29848f3f5205",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2339eb62-6bc2-478d-803b-e56e5fb22844",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install diffusers\n",
+    "!pip install transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3bc31594-7a94-4255-85da-07a5c484e2b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from diffusers import HunyuanVideoPipeline, PipelineQuantizationConfig\n",
+    "import torch\n",
+    "\n",
+    "pipeline = HunyuanVideoPipeline.from_pretrained(\n",
+    "    \"hunyuanvideo-community/HunyuanVideo\",\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    ")\n",
+    "print(pipeline.config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2bfdeca9-ab44-4380-ad3e-c3ec2c2b0d0e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from diffusers import TextToVideoZeroPipeline\n",
+    "\n",
+    "model_id = \"stable-diffusion-v1-5/stable-diffusion-v1-5\"\n",
+    "pipe = TextToVideoZeroPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to(\"cuda\")\n",
+    "print(pipe.config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e7ea86aa-75d0-4dda-8f4a-12a666c34fb2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from diffusers import CogVideoXPipeline\n",
+    "pipe = CogVideoXPipeline.from_pretrained(\"THUDM/CogVideoX-2b\", torch_dtype=torch.float16).to(\"cuda\")\n",
+    "print(pipe.config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e3e6c662-7fea-4474-9ba0-cbaaf5a5cca7",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "af6f3aae-3298-41aa-a9ed-37da79675ab3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python3 (main venv)",
+   "language": "python",
+   "name": "main"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}