chore: export LocalTokenizer to allow usage

.github/workflows/import.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel
+          pip install setuptools wheel sentencepiece protobuf
           pip install pytest
           pip install .
 

google/genai/__init__.py

@@ -18,8 +18,9 @@
 from . import types
 from . import version
 from .client import Client
+from .local_tokenizer import LocalTokenizer
 
 
 __version__ = version.__version__
 
-__all__ = ['Client']
+__all__ = ["Client", "LocalTokenizer"]

google/genai/_local_tokenizer_loader.py

@@ -22,8 +22,8 @@
 import uuid
 
 import requests  # type: ignore
-import sentencepiece as spm
-from sentencepiece import sentencepiece_model_pb2
+from google3.third_party.sentencepiece.src.python import sentencepiece_processor as spm
+from google3.third_party.sentencepiece.src import sentencepiece_model_pb2
 
 
 # Source of truth: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models

google/genai/local_tokenizer.py

@@ -19,7 +19,7 @@
 from typing import Any, Iterable
 from typing import Optional, Union
 
-from sentencepiece import sentencepiece_model_pb2
+from google3.third_party.sentencepiece.src import sentencepiece_model_pb2
 
 from . import _common
 from . import _local_tokenizer_loader as loader

google/genai/tests/local_tokenizer/test_local_tokenizer.py

@@ -18,8 +18,8 @@
 
 from sentencepiece import sentencepiece_model_pb2
 
-from ... import local_tokenizer
-from ... import types
+from ...private import local_tokenizer
+from ...private import types
 
 
 class TestLocalTokenizer(unittest.TestCase):