Skip to main content

​
Embeddings

Every clip and frame has a vector. Query by text or image, find similar content, fetch raw vectors, and visualize the embedding space.

​
Query by text

results = client.embeddings.query(query_text="welding sparks")
for hit in results.items:
    print(hit.entity_id, hit.similarity_score)

​
Query by image

import base64

with open("photo.jpg", "rb") as f:
    b64 = base64.b64encode(f.read()).decode()

results = client.embeddings.query(query_image_base64=b64)

​
Find similar

Given a clip or frame, find the nearest neighbors in embedding space. 
similar = client.embeddings.similar(entity_id="CLIP_ID", entity_type="clip")
for hit in similar.items:
    print(hit.entity_id, hit.similarity_score)

​
Fetch raw vectors

# Single clip
vec = client.embeddings.get_clip_embedding("clip-id")
print(vec.vector_dimensions)  # e.g. 768
print(len(vec.vector))        # 768 floats

# Batch
items = client.embeddings.batch(
    entity_ids=["id1", "id2", "id3"],
    entity_type="clip",
    include_vector=True,
)
vectors = [item.vector for item in items]

​
Visualize: UMAP embedding map

Pull vectors for a set of clips and project them to 2D with UMAP. Color by factory to see how the embedding space clusters. 
pip install buildai-sdk umap-learn matplotlib numpy

import numpy as np
import matplotlib.pyplot as plt
import umap
from buildai import Client

client = Client()

# Grab clips from a few factories
factory_ids = ["factory-1", "factory-2", "factory-3"]
clip_ids = []
clip_factories = []

for fid in factory_ids:
    clips = client.clips.list(factory_id=fid, page_size=50)
    for clip in clips.items:
        clip_ids.append(clip.clip_id)
        clip_factories.append(fid)

# Fetch vectors in batches of 100
vectors = []
for i in range(0, len(clip_ids), 100):
    batch = client.embeddings.batch(
        entity_ids=clip_ids[i : i + 100],
        entity_type="clip",
        include_vector=True,
    )
    vectors.extend([item.vector for item in batch if item.vector])

X = np.array(vectors)

# Project to 2D
reducer = umap.UMAP(n_neighbors=15, min_dist=0.1, metric="cosine")
embedding_2d = reducer.fit_transform(X)

# Plot
fig, ax = plt.subplots(figsize=(10, 8))
for fid in factory_ids:
    mask = [f == fid for f in clip_factories[: len(vectors)]]
    ax.scatter(
        embedding_2d[mask, 0],
        embedding_2d[mask, 1],
        label=fid,
        s=12,
        alpha=0.7,
    )
ax.legend()
ax.set_title("Clip embeddings by factory")
plt.tight_layout()
plt.savefig("embedding_map.png", dpi=150)
plt.show()

​
Cosine similarity matrix

Compare a set of clips pairwise. 
import numpy as np
from buildai import Client

client = Client()

# Get vectors
items = client.embeddings.batch(
    entity_ids=["clip-a", "clip-b", "clip-c", "clip-d"],
    entity_type="clip",
    include_vector=True,
)
X = np.array([item.vector for item in items])

# Normalize and compute cosine similarity
X_norm = X / np.linalg.norm(X, axis=1, keepdims=True)
sim_matrix = X_norm @ X_norm.T

print(sim_matrix)
# [[1.00  0.87  0.34  0.12]
#  [0.87  1.00  0.31  0.15]
#  [0.34  0.31  1.00  0.91]
#  [0.12  0.15  0.91  1.00]]

​
Representation profiles

The representation_profile parameter selects which embedding model to use. Default is "general" which works for most use cases. 
# General-purpose (default)
results = client.embeddings.query(query_text="soldering", representation_profile="general")

# Frame-level
results = client.embeddings.query(query_text="soldering", representation_profile="frame-first")