diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java index 587fa3a8e..6b33c380b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java @@ -41,7 +41,7 @@ public class Hotel { @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List indexedEuclidean; - @VectorStoreRecordData + @VectorStoreRecordData(isFilterable = true) private final List tags; @VectorStoreRecordData diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java index cfff9ea03..93ee1c205 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.data.redis.RedisVectorStore; import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.tests.data.jdbc.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index 513ab72f0..ef4fe9549 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -6,11 +6,10 @@ import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -43,7 +42,6 @@ static class GitHubFile { @VectorStoreRecordData private final String description; @VectorStoreRecordData - @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -125,24 +123,27 @@ public static void inMemoryStoreAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Build a vectorized search - var vectorStoreTextSearch = VectorStoreTextSearch.builder() - .withVectorizedSearch(collection) - .withTextEmbeddingGenerationService(embeddingGeneration) - .build(); - // Search for results // Volatile store executes an exhaustive search, for approximate search use Azure AI Search, Redis or JDBC with PostgreSQL - String query = "How to get started?"; - var results = vectorStoreTextSearch.searchAsync(query, null) - .block(); + var results = search("How to get started", collection, embeddingGeneration).block(); if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 9b2e70acd..382bf0f83 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -5,16 +5,17 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; +import com.azure.core.util.ClientOptions; +import com.azure.core.util.MetricsOptions; +import com.azure.core.util.TracingOptions; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.SearchIndexClientBuilder; -import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreOptions; import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -49,12 +50,11 @@ public class VectorStoreWithAzureAISearch { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @VectorStoreRecordKey() + @VectorStoreRecordKey private final String id; - @VectorStoreRecordData() + @VectorStoreRecordData private final String description; @VectorStoreRecordData - @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List embedding; @@ -64,10 +64,10 @@ public GitHubFile() { } public GitHubFile( - @JsonProperty("fileId") String id, - @JsonProperty("description") String description, - @JsonProperty("link") String link, - @JsonProperty("embedding") List embedding) { + String id, + String description, + String link, + List embedding) { this.id = id; this.description = description; this.link = link; @@ -108,6 +108,7 @@ public static void main(String[] args) { var searchClient = new SearchIndexClientBuilder() .endpoint(AZURE_AI_SEARCH_ENDPOINT) .credential(new AzureKeyCredential(AZURE_AISEARCH_KEY)) + .clientOptions(clientOptions()) .buildAsyncClient(); storeAndSearch(searchClient, embeddingGeneration); @@ -137,24 +138,27 @@ public static void storeAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Build a vectorized search - var vectorStoreTextSearch = VectorStoreTextSearch.builder() - .withVectorizedSearch(collection) - .withTextEmbeddingGenerationService(embeddingGeneration) - .build(); - // Search for results // Might need to wait for the data to be indexed - String query = "How to get started?"; - var results = vectorStoreTextSearch.searchAsync(query, null) - .block(); + var results = search("How to get started", collection, embeddingGeneration).block(); if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( @@ -197,4 +201,11 @@ private static Map sampleData() { "README: README associated with a sample chat summary react-based webapp" }, }).collect(Collectors.toMap(element -> element[0], element -> element[1])); } + + private static ClientOptions clientOptions() { + return new ClientOptions() + .setTracingOptions(new TracingOptions()) + .setMetricsOptions(new MetricsOptions()) + .setApplicationId("Semantic-Kernel"); + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 3a393ad29..b9ff7aa91 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -9,15 +9,14 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; -import com.mysql.cj.jdbc.MysqlDataSource; + import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.util.Arrays; @@ -27,6 +26,7 @@ import java.util.Map; import java.util.stream.Collectors; +import org.postgresql.ds.PGSimpleDataSource; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -42,12 +42,11 @@ public class VectorStoreWithJDBC { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @VectorStoreRecordKey() + @VectorStoreRecordKey private final String id; - @VectorStoreRecordData() + @VectorStoreRecordData private final String description; @VectorStoreRecordData - @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -89,8 +88,8 @@ static String encodeId(String realId) { } } - // Run a MySQL server with: - // docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest + // Run a PostgreSQL server with: + // docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17 public static void main(String[] args) throws SQLException { System.out.println("=============================================================="); @@ -123,14 +122,14 @@ public static void main(String[] args) throws SQLException { public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { // Configure the data source - var dataSource = new MysqlDataSource(); - dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); dataSource.setPassword("root"); - dataSource.setUser("root"); // Build a query provider // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider - var queryProvider = MySQLVectorStoreQueryProvider.builder() + var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); @@ -155,23 +154,26 @@ public static void storeAndSearch(OpenAITextEmbeddingGenerationService embedding .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Build a vectorized search - var vectorStoreTextSearch = VectorStoreTextSearch.builder() - .withVectorizedSearch(collection) - .withTextEmbeddingGenerationService(embeddingGeneration) - .build(); - // Search for results - String query = "How to get started?"; - var results = vectorStoreTextSearch.searchAsync(query, null) - .block(); + var results = search("How to get started", collection, embeddingGeneration).block(); if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index b037f6f74..b6bebff81 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -10,9 +10,8 @@ import com.microsoft.semantickernel.data.redis.RedisStorageType; import com.microsoft.semantickernel.data.redis.RedisVectorStore; import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -42,12 +41,11 @@ public class VectorStoreWithRedis { private static final int EMBEDDING_DIMENSIONS = 1536; public static class GitHubFile { - @VectorStoreRecordKey() + @VectorStoreRecordKey private final String id; - @VectorStoreRecordData() + @VectorStoreRecordData private final String description; @VectorStoreRecordData - @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -146,23 +144,27 @@ public static void storeAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Build a vectorized search - var vectorStoreTextSearch = VectorStoreTextSearch.builder() - .withVectorizedSearch(collection) - .withTextEmbeddingGenerationService(embeddingGeneration) - .build(); - // Search for results - String query = "How to get started?"; - var results = vectorStoreTextSearch.searchAsync(query, null) - .block(); + // Might need to wait for the data to be indexed + var results = search("How to get started", collection, embeddingGeneration).block(); if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 6978bdeee..d788980e4 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -29,6 +29,19 @@ semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-data-azureaisearch + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-data-redis + + org.apache.logging.log4j log4j-api diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Hotel.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Hotel.java new file mode 100644 index 000000000..6fff7664f --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Hotel.java @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.index; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.util.Collections; +import java.util.List; + +public class Hotel { + @VectorStoreRecordKey + private String hotelId; + + @VectorStoreRecordData(isFilterable = true) + private String name; + + @VectorStoreRecordData(isFullTextSearchable = true) + private String description; + + @VectorStoreRecordVector(dimensions = 4, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + private List descriptionEmbedding; + + @VectorStoreRecordData(isFilterable = true) + private List tags; + + public Hotel() { + } + + public Hotel(String hotelId, String name, String description, List descriptionEmbedding, + List tags) { + this.hotelId = hotelId; + this.name = name; + this.description = description; + this.descriptionEmbedding = Collections.unmodifiableList(descriptionEmbedding); + this.tags = Collections.unmodifiableList(tags); + } + + public String getHotelId() { + return hotelId; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public List getDescriptionEmbedding() { + return descriptionEmbedding; + } + + public List getTags() { + return tags; + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Main.java new file mode 100644 index 000000000..414c9be8c --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/index/Main.java @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.index; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.postgresql.ds.PGSimpleDataSource; +import reactor.core.publisher.Mono; + +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Create a PostgreSQL data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Create a JDBC vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build()) + .build(); + + // Get a collection from the vector store + var collection = vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionAsync().block(); + + // Upsert a record. + var description = "A place where everyone can be happy"; + var hotelId = "1"; + var hotel = new Hotel( + hotelId, + "Hotel Happy", + description, + generateEmbeddingsAsync(description).block(), + List.of("luxury", "pool")); + + collection.upsertAsync(hotel, null).block(); + + // Retrieve the upserted record. + var retrievedHotel = collection.getAsync(hotelId, null).block(); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. + var searchVector = generateEmbeddingsAsync( + "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search. + var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() + .withTop(1).build()).block(); + + Hotel record = searchResult.getResults().get(0).getRecord(); + System.out.printf("Found hotel description: %s\n", record.getDescription()); + } + + private static Mono> generateEmbeddingsAsync(String text) { + return Mono.just(List.of(1.0f, 2.0f, 3.0f, 4.0f)); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/recorddefinition/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/recorddefinition/Main.java new file mode 100644 index 000000000..cf6ad56ba --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/recorddefinition/Main.java @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.recorddefinition; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.*; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import org.postgresql.ds.PGSimpleDataSource; + +import java.util.Arrays; +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Create a PostgreSQL data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Create a JDBC vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build()) + .build(); + + var hotelDefinition = VectorStoreRecordDefinition.fromFields( + Arrays.asList( + VectorStoreRecordKeyField.builder().withName("hotelId").withFieldType(String.class) + .build(), + VectorStoreRecordDataField.builder() + .withName("name") + .withFieldType(String.class) + .isFilterable(true).build(), + VectorStoreRecordDataField.builder() + .withName("description") + .withFieldType(String.class) + .isFullTextSearchable(true).build(), + VectorStoreRecordVectorField.builder().withName("descriptionEmbedding") + .withDimensions(4) + .withIndexKind(IndexKind.HNSW) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withFieldType(List.class).build())); + + var collection = vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordDefinition(hotelDefinition) + .withRecordClass(Hotel.class) + .build()); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/hotels/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/hotels/Main.java new file mode 100644 index 000000000..a53562442 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/hotels/Main.java @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorsearch.hotels; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import org.postgresql.ds.PGSimpleDataSource; +import reactor.core.publisher.Mono; + +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Configure the data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Create a JDBC vector store and choose an existing collection that already contains records. + var vectorStore = new JDBCVectorStore(dataSource, JDBCVectorStoreOptions.builder() + .withQueryProvider(PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build()); + var collection = vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. + var searchVector = generateEmbeddingsAsync( + "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search, passing an options object with a Top value to limit results to the single top match. + var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() + .withTop(1).build()).block(); + + // Inspect the returned hotel. + Hotel hotel = searchResult.getResults().get(0).getRecord(); + System.out.printf("Found hotel description: %s\n", hotel.getDescription()); + } + + private static Mono> generateEmbeddingsAsync(String text) { + return Mono.just(List.of(1.0f, 2.0f, 3.0f, 4.0f)); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/products/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/products/Main.java new file mode 100644 index 000000000..f96593284 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorsearch/products/Main.java @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorsearch.products; + +import com.microsoft.semantickernel.data.VolatileVectorStore; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Build a query provider + var vectorStore = new VolatileVectorStore(); + var collection = vectorStore.getCollection("skproducts", + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Product.class) + .build()); + collection.createCollectionIfNotExistsAsync().block(); + var vector = generateEmbeddingsAsync().block(); + collection.upsertAsync( + new Product("1", "Product 1", List.of("Feature 1", "Feature 2"), vector, vector), null) + .block(); + + withVectorFieldName(collection); + withTopAndSkip(collection); + withIncludeVectors(collection); + withVectorSearchFilter(); + } + + public static void withVectorFieldName( + VectorStoreRecordCollection collection) { + // Create the vector search options and indicate that we want to search the FeatureListEmbedding field. + var searchOptions = VectorSearchOptions.builder() + .withVectorFieldName("featureListEmbedding") + .build(); + + // Generate a vector for your search text, using the embedding model of your choice + var searchVector = generateEmbeddingsAsync().block(); + + // Do the search + var searchResult = collection.searchAsync(searchVector, searchOptions).block(); + } + + public static void withTopAndSkip(VectorStoreRecordCollection collection) { + // Create the vector search options and indicate that we want to skip the first 40 results and then get the next 20. + var searchOptions = VectorSearchOptions.builder() + .withTop(20) + .withSkip(40) + .build(); + + // Generate a vector for your search text, using the embedding model of your choice + var searchVector = generateEmbeddingsAsync().block(); + + // Do the search + var searchResult = collection.searchAsync(searchVector, searchOptions).block(); + } + + public static void withIncludeVectors(VectorStoreRecordCollection collection) { + // Create the vector search options and indicate that we want to include vectors in the search results. + var searchOptions = VectorSearchOptions.builder() + .withIncludeVectors(true) + .build(); + + // Generate a vector for your search text, using the embedding model of your choice + var searchVector = generateEmbeddingsAsync().block(); + + // Do the search + var searchResult = collection.searchAsync(searchVector, searchOptions).block(); + } + + public static void withVectorSearchFilter() { + // Build a query provider + var vectorStore = new VolatileVectorStore(); + var collection = vectorStore.getCollection("skglossary", + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Glossary.class) + .build()); + collection.createCollectionIfNotExistsAsync().block(); + var vector = generateEmbeddingsAsync().block(); + collection.upsertAsync(new Glossary("1", "External Definitions", List.of("memory"), + "Memory", "The power of the mind to remember things", vector), null).block(); + + // Filter where category == 'External Definitions' and tags contain 'memory'. + var filter = VectorSearchFilter.builder() + .equalTo("category", "External Definitions") + .anyTagEqualTo("tags", "memory") + .build(); + + // Create the vector search options and indicate that we want to filter the search results by a specific field. + var searchOptions = VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build(); + + // Generate a vector for your search text, using the embedding model of your choice + var searchVector = generateEmbeddingsAsync().block(); + + // Do the search + var searchResult = collection.searchAsync(searchVector, searchOptions).block(); + } + + public static class Product { + @VectorStoreRecordKey + private String key; + + @VectorStoreRecordData + private String description; + + @VectorStoreRecordData + private List featureList; + + @VectorStoreRecordVector(dimensions = 1536) + public List descriptionEmbedding; + + @VectorStoreRecordVector(dimensions = 1536) + public List featureListEmbedding; + + public Product() { + } + + public Product(String key, String description, List featureList, + List descriptionEmbedding, List featureListEmbedding) { + this.key = key; + this.description = description; + this.featureList = featureList; + this.descriptionEmbedding = Collections.unmodifiableList(descriptionEmbedding); + this.featureListEmbedding = Collections.unmodifiableList(featureListEmbedding); + } + + public String getKey() { + return key; + } + + public String getDescription() { + return description; + } + + public List getFeatureList() { + return featureList; + } + + public List getDescriptionEmbedding() { + return descriptionEmbedding; + } + + public List getFeatureListEmbedding() { + return featureListEmbedding; + } + } + + public static class Glossary { + @VectorStoreRecordKey + private String key; + + @VectorStoreRecordData(isFilterable = true) + private String category; + + @VectorStoreRecordData(isFilterable = true) + private List tags; + + @VectorStoreRecordData + private String term; + + @VectorStoreRecordData + private String definition; + + @VectorStoreRecordVector(dimensions = 1536) + private List definitionEmbedding; + + public Glossary() { + } + + public Glossary(String key, String category, List tags, String term, + String definition, List definitionEmbedding) { + this.key = key; + this.category = category; + this.tags = tags; + this.term = term; + this.definition = definition; + this.definitionEmbedding = Collections.unmodifiableList(definitionEmbedding); + } + + public String getKey() { + return key; + } + + public String getCategory() { + return category; + } + + public List getTags() { + return tags; + } + + public String getTerm() { + return term; + } + + public String getDefinition() { + return definition; + } + + public List getDefinitionEmbedding() { + return definitionEmbedding; + } + } + + private static Mono> generateEmbeddingsAsync() { + return Mono.just(new ArrayList<>(Collections.nCopies(1536, 1.0f))); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/azureaisearch/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/azureaisearch/Main.java new file mode 100644 index 000000000..89ce88ea3 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/azureaisearch/Main.java @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.azureaisearch; + +import com.azure.core.credential.AzureKeyCredential; +import com.azure.search.documents.indexes.SearchIndexClientBuilder; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStore; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Build the Azure AI Search client + var searchClient = new SearchIndexClientBuilder() + .endpoint("https://.search.windows.net") + .credential(new AzureKeyCredential("")) + .buildAsyncClient(); + + // Build an Azure AI Search Vector Store + var vectorStore = AzureAISearchVectorStore.builder() + .withSearchIndexAsyncClient(searchClient) + .withOptions(new AzureAISearchVectorStoreOptions()) + .build(); + + var collection = new AzureAISearchVectorStoreRecordCollection<>(searchClient, "skhotels", + AzureAISearchVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/inmemory/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/inmemory/Main.java new file mode 100644 index 000000000..fff6e115a --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/inmemory/Main.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.inmemory; + +import com.microsoft.semantickernel.data.VolatileVectorStore; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Build an Azure AI Search Vector Store + var vectorStore = new VolatileVectorStore(); + + var collection = new VolatileVectorStoreRecordCollection<>("skhotels", + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/jdbc/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/jdbc/Main.java new file mode 100644 index 000000000..8b7bc1257 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/jdbc/Main.java @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.jdbc; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import org.postgresql.ds.PGSimpleDataSource; + +public class Main { + public static void main(String[] args) { + // Configure the data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Build a query provider + // Other available query providers are MySQLVectorStoreQueryProvider, SQLiteVectorStoreQueryProvider + // and HSQDBVectorStoreQueryProvider + var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + var collection = new JDBCVectorStoreRecordCollection<>( + dataSource, + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/redis/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/redis/Main.java new file mode 100644 index 000000000..418c587c9 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/redis/Main.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.redis; + +import com.microsoft.semantickernel.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisStorageType; +import com.microsoft.semantickernel.data.redis.RedisVectorStore; +import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import redis.clients.jedis.JedisPooled; + +public class Main { + public static void main(String[] args) { + JedisPooled jedis = new JedisPooled(""); + + // Build a Redis Vector Store + // Available storage types are JSON and HASHSET. Default is JSON. + var vectorStore = RedisVectorStore.builder() + .withClient(jedis) + .withOptions( + RedisVectorStoreOptions.builder() + .withStorageType(RedisStorageType.HASH_SET).build()) + .build(); + + var collection = vectorStore.getCollection("skhotels", + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + collection = vectorStore.getCollection("skhotels", + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withPrefixCollectionName(false) + .build()); + + collection.getAsync("myprefix_h1", null).block(); + } +}