Semantic Search Workflow Fresh 🌱

Build a cross-modal semantic search system that finds relevant content across text, images, video, and audio.

Search Architecture

Cross-Modal Search Flow

Implementation

Step 1: Build the Index

from google import genai
from google.genai import types
import numpy as np
import json

client = genai.Client()
DIMENSIONS = 768  # Cost-effective for search

class SearchIndex:
    def __init__(self):
        self.embeddings = []
        self.metadata = []

    def add_text(self, doc_id, text):
        result = client.models.embed_content(
            model='gemini-embedding-2-preview',
            contents=text,
            config=types.EmbedContentConfig(
                task_type='RETRIEVAL_DOCUMENT',
                output_dimensionality=DIMENSIONS
            )
        )
        vec = result.embeddings[0].values
        # Normalize for reduced dimensions
        vec = vec / np.linalg.norm(vec)
        self.embeddings.append(vec)
        self.metadata.append({'id': doc_id, 'type': 'text', 'content': text[:200]})

    def add_image(self, doc_id, image_path, caption=''):
        with open(image_path, 'rb') as f:
            result = client.models.embed_content(
                model='gemini-embedding-2-preview',
                contents=[types.Part.from_bytes(
                    data=f.read(), mime_type='image/jpeg'
                )],
                config=types.EmbedContentConfig(
                    output_dimensionality=DIMENSIONS
                )
            )
        vec = result.embeddings[0].values
        vec = vec / np.linalg.norm(vec)
        self.embeddings.append(vec)
        self.metadata.append({
            'id': doc_id, 'type': 'image',
            'path': image_path, 'caption': caption
        })

    def add_audio(self, doc_id, audio_path, description=''):
        with open(audio_path, 'rb') as f:
            result = client.models.embed_content(
                model='gemini-embedding-2-preview',
                contents=[types.Part.from_bytes(
                    data=f.read(), mime_type='audio/mp3'
                )],
                config=types.EmbedContentConfig(
                    output_dimensionality=DIMENSIONS
                )
            )
        vec = result.embeddings[0].values
        vec = vec / np.linalg.norm(vec)
        self.embeddings.append(vec)
        self.metadata.append({
            'id': doc_id, 'type': 'audio',
            'path': audio_path, 'description': description
        })

    def search(self, query_text, top_k=10):
        """Search the index with a text query."""
        result = client.models.embed_content(
            model='gemini-embedding-2-preview',
            contents=query_text,
            config=types.EmbedContentConfig(
                task_type='RETRIEVAL_QUERY',
                output_dimensionality=DIMENSIONS
            )
        )
        query_vec = np.array(result.embeddings[0].values)
        query_vec = query_vec / np.linalg.norm(query_vec)

        # Compute cosine similarities
        matrix = np.array(self.embeddings)
        scores = matrix @ query_vec

        # Rank by score
        top_indices = np.argsort(scores)[::-1][:top_k]
        results = []
        for idx in top_indices:
            results.append({
                **self.metadata[idx],
                'score': float(scores[idx])
            })
        return results

Step 2: Index Content

index = SearchIndex()

# Index text documents
index.add_text('doc1', 'Machine learning is a subset of artificial intelligence...')
index.add_text('doc2', 'The sunset painted the sky in shades of orange and purple...')

# Index images
index.add_image('img1', 'photos/beach_sunset.jpg', 'Beach sunset')
index.add_image('img2', 'photos/city_skyline.jpg', 'City at night')

# Index audio
index.add_audio('aud1', 'audio/ocean_waves.mp3', 'Ocean waves')
index.add_audio('aud2', 'audio/birdsong.mp3', 'Morning birdsong')

print(f"Indexed {len(index.embeddings)} items")

Step 3: Search Across Modalities

results = index.search('relaxing nature sounds', top_k=5)

for r in results:
    print(f"[{r['type']}] {r['id']} — score: {r['score']:.4f}")
    if r['type'] == 'text':
        print(f"  Content: {r['content']}")
    elif r['type'] == 'image':
        print(f"  Caption: {r['caption']}")
    elif r['type'] == 'audio':
        print(f"  Description: {r['description']}")

Step 4: Production Setup with Qdrant

from qdrant_client import QdrantClient
from qdrant_client.models import VectorParams, Distance, PointStruct

# Initialize Qdrant
qdrant = QdrantClient(host='localhost', port=6333)

# Create collection
qdrant.create_collection(
    collection_name='multimodal_search',
    vectors_config=VectorParams(
        size=768,
        distance=Distance.COSINE
    )
)

# Insert vectors
points = []
for i, (emb, meta) in enumerate(zip(index.embeddings, index.metadata)):
    points.append(PointStruct(
        id=i,
        vector=emb.tolist(),
        payload=meta
    ))

qdrant.upsert(collection_name='multimodal_search', points=points)

# Search
query_result = client.models.embed_content(
    model='gemini-embedding-2-preview',
    contents='beautiful landscape',
    config=types.EmbedContentConfig(
        task_type='RETRIEVAL_QUERY',
        output_dimensionality=768
    )
)

hits = qdrant.search(
    collection_name='multimodal_search',
    query_vector=query_result.embeddings[0].values,
    limit=10
)

for hit in hits:
    print(f"Score: {hit.score:.4f} | {hit.payload}")

Verification Checklist

• [ ] Index built with documents across multiple modalities
• [ ] Text queries returning relevant cross-modal results
• [ ] Normalization applied for reduced dimensions
• [ ] RETRIEVAL_DOCUMENT used for indexing, RETRIEVAL_QUERY for search
• [ ] Similarity scores meaningful (higher = more relevant)
• [ ] Production vector database integration tested

Troubleshooting

Issue	Solution
All scores near zero	Ensure normalization for dimensions below 3072
No cross-modal results	Verify same model and dimensions for index and query
Slow search	Use approximate nearest neighbor (ANN) index in production
Poor relevance	Use task types; try higher dimensions (1536, 3072)
Memory issues	Use streaming insertion for large datasets

See Also

• RAG Pipeline — Search + generation
• Multimodal Embeddings — Interleaved input
• Integrations — Vector DB integrations
• Benchmarks — Quality by dimension