Azure AI Search with vectorized search
We can create a vectorized search for our Azure AI Search without going through setting up Azure Foundry embedding model. We will just use standard embedding "hnsw" and we also don't require a indexer for now, probably when we have more document to index.
The setup process would be
1. Create index
2. Embed and upload your docs - we require this to show how we can vectorized our document so we can test it out later
3. Perform vector search
Creating index
We can create our vector index called "index-vector" using the following code. As you can see here, we are also embedding and uploading the document
from azure.identity import DefaultAzureCredential
from azure.core.credentials import AzureKeyCredential
from azure.search.documents.indexes import SearchIndexClient
from azure.search.documents import SearchClient
from azure.search.documents.indexes.models import (
ComplexField,
SimpleField,
SearchFieldDataType,
SearchableField,
SearchIndex,
SearchField,
VectorSearch,
HnswAlgorithmConfiguration,
VectorSearchProfile,
)
from azure.search.documents import SearchClient
from sentence_transformers import SentenceTransformer
from azure.search.documents.models import VectorizedQuery
credential = DefaultAzureCredential()
index_name: str = "index-vector"
search_endpoint: str = "https://your-search-endpoint.search.windows.net"
index_client = SearchIndexClient(
endpoint=search_endpoint, credential=credential)
api_key = "your-api-key"
# Delete old index
try:
index_client.delete_index(index_name)
print(f"Deleted old index '{index_name}'")
except:
pass
# Create new index with retrievable embedding field
fields = [
SimpleField(name="id", type=SearchFieldDataType.String, key=True),
SearchableField(name="text", type=SearchFieldDataType.String, retrievable=True),
SearchableField(name="source", type=SearchFieldDataType.String, retrievable=True),
SearchField(
name="embedding",
type=SearchFieldDataType.Collection(SearchFieldDataType.Single),
searchable=True,
retrievable=True, # KEY: Make embedding retrievable
vector_search_dimensions=384,
vector_search_profile_name="myHnsw",
),
]
vector_search_config = VectorSearch(
algorithms=[HnswAlgorithmConfiguration(name="myHnsw")],
profiles=[VectorSearchProfile(name="myHnsw", algorithm_configuration_name="myHnsw")],
)
index = SearchIndex(
name=index_name,
fields=fields,
vector_search=vector_search_config,
)
index_client.create_index(index)
print(f"✓ Created index '{index_name}' with retrievable embedding field")
# Now re-upload the hotel documents
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("all-MiniLM-L6-v2")
sample_documents = [
{
"id": "hotel-001",
"text": "Sunset Beach Resort is a luxury 5-star hotel located in Miami Beach with oceanfront views. Features include a spa, fine dining restaurants, and a private beach access.",
"source": "hotel-booking-miami.txt"
},
{
"id": "hotel-002",
"text": "Mountain View Lodge is a cozy 3-star hotel nestled in the Rocky Mountains near Denver. Perfect for hiking enthusiasts.",
"source": "hotel-booking-denver.txt"
},
{
"id": "hotel-003",
"text": "Downtown Urban Hotel is a business-oriented 4-star property in the heart of New York City.",
"source": "hotel-booking-nyc.txt"
},
{
"id": "hotel-004",
"text": "Tropical Paradise Resort in Cancun offers all-inclusive packages with beachfront bungalows and water sports.",
"source": "hotel-booking-cancun.txt"
},
{
"id": "hotel-005",
"text": "Historic Inn and Spa in Charleston features charming colonial architecture and luxury spa treatments.",
"source": "hotel-booking-charleston.txt"
}
]
documents_to_update = []
for doc in sample_documents:
doc_with_embedding = {
"id": doc["id"],
"text": doc["text"],
"source": doc["source"],
"embedding": model.encode(doc["text"]).tolist()
}
documents_to_update.append(doc_with_embedding)
from azure.search.documents import SearchClient
search_client = SearchClient(
endpoint=search_endpoint,
credential=AzureKeyCredential(api_key),
index_name=index_name
)
search_client.upload_documents(documents_to_update)
print(f"✓ Uploaded {len(documents_to_update)} hotel documents\n")
You can see that we have 5 document added into our index.
Then we will setup our search
from azure.identity import DefaultAzureCredential
from azure.core.credentials import AzureKeyCredential
from azure.search.documents.indexes import SearchIndexClient, SearchIndexerClient
from azure.search.documents import SearchClient
from azure.search.documents.indexes.models import (
ComplexField,
SimpleField,
SearchFieldDataType,
SearchableField,
SearchIndex,
SearchField,
VectorSearch,
HnswAlgorithmConfiguration,
VectorSearchProfile,
)
from azure.search.documents import SearchClient
from sentence_transformers import SentenceTransformer
from azure.search.documents.models import VectorizedQuery
credential = DefaultAzureCredential()
search_endpoint: str = "https://your-search-endpoint.search.windows.net"
api_key = "your-api-key" # Replace with your actual API key
# Load the embedding model (produces 384-dimensional vectors)
model = SentenceTransformer("all-MiniLM-L6-v2")
# Get status of a specific indexer
index_name: str = "index-vector"
index_client = SearchIndexClient(
endpoint=search_endpoint, credential=credential)
indexes = index_client.list_index_names()
print(f"Available indexes: {list(indexes)}")
# Check how many documents are actually in your index
from azure.search.documents import SearchClient
search_client = SearchClient(
endpoint=search_endpoint,
credential=AzureKeyCredential(api_key),
index_name=index_name
)
# Count documents in index
results = search_client.search(search_text="*", select=["id"])
doc_count = sum(1 for _ in results)
print(f"\nDocuments in index: {doc_count}")
# setup index search
search_client = SearchClient(
endpoint=search_endpoint,
credential=AzureKeyCredential(api_key),
index_name=index_name
)
def search_vector(question, k=3):
"""Pure vector search"""
query_vector = model.encode(question)
vector_query = VectorizedQuery(
vector=query_vector.tolist(),
k_nearest_neighbors=k,
fields="embedding",
)
results = search_client.search(
search_text=None,
vector_queries=[vector_query],
select=["id", "text", "source"], # Fixed field names
)
return results
def search_hybrid(question, k=3):
query_vector = model.encode(question)
vector_query = VectorizedQuery(
vector=query_vector.tolist(),
k_nearest_neighbors=k,
fields="embedding",
)
results = search_client.search(
search_text=question, # Keyword search too
vector_queries=[vector_query],
select=["id", "text", "source"],
top=k,
)
return results
# Perform search works!
search_term = "hotel"
print(f"=== Hybrid Search Results for '{search_term}' ===")
query_vector = model.encode(search_term)
from azure.search.documents.models import VectorizedQuery
vector_query = VectorizedQuery(
vector=query_vector.tolist(),
k_nearest_neighbors=5,
fields="embedding",
)
results = search_client.search(
search_text=search_term,
vector_queries=[vector_query],
select=["id", "text", "source", "embedding"],
top=5,
)
for result in results:
print(f"\nScore: {result['@search.score']:.4f}")
print(f"Source: {result['source']}")
print(f"Text: {result['text'][:100]}...")
And let's test it out by passing in and not so relevant text like "unknownforsure" and you can see the score is pretty low
And when we search for "hotel", our score become higher:-
This is what our index vector in json looks like.
{
"@odata.etag": "\"0x8DF06160AB01F08\"",
"name": "index-vector",
"purviewEnabled": false,
"fields": [
{
"name": "id",
"type": "Edm.String",
"searchable": false,
"filterable": false,
"retrievable": true,
"stored": true,
"sortable": false,
"facetable": false,
"key": true,
"synonymMaps": []
},
{
"name": "text",
"type": "Edm.String",
"searchable": true,
"filterable": false,
"retrievable": true,
"stored": true,
"sortable": false,
"facetable": false,
"key": false,
"synonymMaps": []
},
{
"name": "source",
"type": "Edm.String",
"searchable": true,
"filterable": false,
"retrievable": true,
"stored": true,
"sortable": false,
"facetable": false,
"key": false,
"synonymMaps": []
},
{
"name": "embedding",
"type": "Collection(Edm.Single)",
"searchable": true,
"filterable": false,
"retrievable": true,
"stored": true,
"sortable": false,
"facetable": false,
"key": false,
"dimensions": 384,
"vectorSearchProfile": "myHnsw",
"synonymMaps": []
}
],
"scoringProfiles": [],
"suggesters": [],
"analyzers": [],
"normalizers": [],
"tokenizers": [],
"tokenFilters": [],
"charFilters": [],
"similarity": {
"@odata.type": "#Microsoft.Azure.Search.BM25Similarity"
},
"vectorSearch": {
"algorithms": [
{
"name": "myHnsw",
"kind": "hnsw",
"hnswParameters": {
"metric": "cosine",
"m": 4,
"efConstruction": 400,
"efSearch": 500
}
}
],
"profiles": [
{
"name": "myHnsw",
"algorithm": "myHnsw"
}
],
"vectorizers": [],
"compressions": []
}
}
Comments