Basic Usage
This guide covers the fundamental concepts and common workflows for using UESynth effectively in your computer vision and machine learning projects.
Core Concepts
1. Client Types
UESynth provides two client types for different use cases:
# Synchronous client - simple, blocking operations
from uesynth import UESynthClient
client = UESynthClient()
# Asynchronous client - high-performance, non-blocking
from uesynth import AsyncUESynthClient
client = AsyncUESynthClient()When to use each:
Sync Client: Prototyping, simple scripts, single-shot captures
Async Client: Real-time simulations, batch processing, high-throughput scenarios
2. Data Modalities
UESynth can capture multiple types of data from your Unreal Engine scene:
# RGB images - standard color images
rgb_image = client.capture.rgb()
# Depth maps - distance from camera to objects
depth_map = client.capture.depth()
# Segmentation masks - object instance IDs
segmentation = client.capture.segmentation()
# Surface normals - object surface orientations
normals = client.capture.normals()
# All at once - efficient multi-modal capture
data = client.capture.all_modalities()Common Workflows
1. Single Image Capture
The simplest workflow - capture one image:
from uesynth import UESynthClient
import cv2
with UESynthClient() as client:
# Position camera
client.camera.set_location(x=0, y=100, z=50)
client.camera.set_rotation(pitch=-15, yaw=0, roll=0)
# Capture and save
image = client.capture.rgb(width=1920, height=1080)
cv2.imwrite("captured_image.png", image)2. Multi-View Capture
Capture the same scene from multiple viewpoints:
from uesynth import UESynthClient
import cv2
def multi_view_capture():
with UESynthClient() as client:
# Define viewpoints
viewpoints = [
{"pos": (0, 200, 100), "rot": (0, 0, 0), "name": "front"},
{"pos": (200, 0, 100), "rot": (0, 90, 0), "name": "right"},
{"pos": (0, -200, 100), "rot": (0, 180, 0), "name": "back"},
{"pos": (-200, 0, 100), "rot": (0, -90, 0), "name": "left"},
{"pos": (0, 0, 300), "rot": (-90, 0, 0), "name": "top"}
]
for i, view in enumerate(viewpoints):
# Position camera
client.camera.set_location(*view["pos"])
client.camera.set_rotation(*view["rot"])
# Capture RGB and depth
rgb = client.capture.rgb()
depth = client.capture.depth()
# Save with descriptive names
cv2.imwrite(f"view_{view['name']}_rgb.png", rgb)
# Normalize depth for visualization
depth_norm = cv2.normalize(depth, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
cv2.imwrite(f"view_{view['name']}_depth.png", depth_norm)
print(f"Captured view {i+1}/{len(viewpoints)}: {view['name']}")
multi_view_capture()3. Object Manipulation Workflow
Move objects around and capture the results:
from uesynth import UESynthClient
import cv2
def object_manipulation_dataset():
with UESynthClient() as client:
# Get list of objects in scene
objects = client.objects.list_all()
print(f"Found {len(objects)} objects in scene")
# Find a specific object (e.g., a car)
car_objects = [obj for obj in objects if "car" in obj.name.lower()]
if not car_objects:
print("No car objects found!")
return
car_name = car_objects[0].name
print(f"Using object: {car_name}")
# Position camera to view the scene
client.camera.set_location(x=0, y=300, z=150)
client.camera.set_rotation(pitch=-20, yaw=0, roll=0)
# Generate dataset with object in different positions
positions = [
(0, 0, 0), # Center
(100, 0, 0), # Right
(-100, 0, 0), # Left
(0, 100, 0), # Forward
(0, -100, 0), # Backward
]
for i, (x, y, z) in enumerate(positions):
# Move object
client.objects.set_location(car_name, x=x, y=y, z=z)
# Capture all modalities
data = client.capture.all_modalities(width=1024, height=768)
# Save each modality
cv2.imwrite(f"car_pos_{i}_rgb.png", data['rgb'])
cv2.imwrite(f"car_pos_{i}_depth.png",
cv2.normalize(data['depth'], None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U))
cv2.imwrite(f"car_pos_{i}_seg.png", data['segmentation'].astype('uint8'))
print(f"Captured position {i+1}/{len(positions)}")
object_manipulation_dataset()4. Time-of-Day Dataset
Create a dataset with different lighting conditions:
from uesynth import UESynthClient
import cv2
def time_of_day_dataset():
with UESynthClient() as client:
# Position camera
client.camera.set_location(x=0, y=200, z=100)
client.camera.set_rotation(pitch=-15, yaw=0, roll=0)
# Different times of day
time_settings = [
{"hour": 6, "name": "dawn"},
{"hour": 9, "name": "morning"},
{"hour": 12, "name": "noon"},
{"hour": 15, "name": "afternoon"},
{"hour": 18, "name": "sunset"},
{"hour": 21, "name": "night"}
]
for time_setting in time_settings:
# Set time of day
client.scene.set_time_of_day(time_setting["hour"])
# Capture image
image = client.capture.rgb(width=1920, height=1080)
# Save with time label
filename = f"scene_{time_setting['name']}.png"
cv2.imwrite(filename, image)
print(f"Captured {time_setting['name']} scene")
time_of_day_dataset()Data Processing Patterns
1. Real-time Processing
Process frames as they're captured:
import cv2
import numpy as np
from uesynth import UESynthClient
def real_time_processing():
with UESynthClient() as client:
# Setup camera
client.camera.set_location(x=0, y=100, z=50)
for i in range(100):
# Move camera slightly each frame
client.camera.set_location(x=i*2, y=100, z=50)
# Capture frame
frame = client.capture.rgb(width=640, height=480)
# Process immediately (example: edge detection)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
edges = cv2.Canny(gray, 100, 200)
# Save processed result
cv2.imwrite(f"processed_frame_{i:03d}.png", edges)
print(f"Processed frame {i+1}/100")
real_time_processing()2. Batch Processing
Collect data first, then process:
import cv2
import numpy as np
from uesynth import UESynthClient
def batch_processing():
# Step 1: Collect all data
print("Collecting data...")
frames = []
with UESynthClient() as client:
client.camera.set_location(x=0, y=100, z=50)
for i in range(50):
client.camera.set_rotation(pitch=-15, yaw=i*7.2, roll=0) # 360° rotation
frame = client.capture.rgb(width=512, height=512)
frames.append(frame)
print(f"Collected {len(frames)} frames")
# Step 2: Process all frames
print("Processing data...")
processed_frames = []
for i, frame in enumerate(frames):
# Example processing: blur and resize
blurred = cv2.GaussianBlur(frame, (15, 15), 0)
resized = cv2.resize(blurred, (256, 256))
processed_frames.append(resized)
if i % 10 == 0:
print(f"Processed {i+1}/{len(frames)} frames")
# Step 3: Save results
print("Saving results...")
for i, processed_frame in enumerate(processed_frames):
cv2.imwrite(f"batch_processed_{i:03d}.png", processed_frame)
batch_processing()3. Dataset Creation
Create structured datasets for ML training:
import cv2
import json
import os
from uesynth import UESynthClient
def create_ml_dataset():
# Create dataset directory structure
os.makedirs("dataset/images", exist_ok=True)
os.makedirs("dataset/annotations", exist_ok=True)
dataset_info = {
"images": [],
"annotations": [],
"metadata": {
"created_with": "UESynth",
"capture_settings": {
"width": 1024,
"height": 768
}
}
}
with UESynthClient() as client:
# Get scene objects for annotations
objects = client.objects.list_all()
for i in range(100):
# Random camera position
import random
x = random.uniform(-200, 200)
y = random.uniform(-200, 200)
z = random.uniform(50, 150)
client.camera.set_location(x=x, y=y, z=z)
client.camera.set_rotation(pitch=-15, yaw=random.uniform(0, 360), roll=0)
# Capture data
rgb = client.capture.rgb(width=1024, height=768)
segmentation = client.capture.segmentation(width=1024, height=768)
# Save image
image_filename = f"image_{i:06d}.png"
cv2.imwrite(f"dataset/images/{image_filename}", rgb)
# Create annotation
annotation = {
"image_id": i,
"filename": image_filename,
"camera_position": {"x": x, "y": y, "z": z},
"objects": []
}
# Analyze segmentation for object info
unique_ids = np.unique(segmentation)
for obj_id in unique_ids:
if obj_id > 0: # Skip background
mask = (segmentation == obj_id)
if mask.sum() > 100: # Only include objects with enough pixels
# Find bounding box
coords = np.where(mask)
y_min, y_max = coords[0].min(), coords[0].max()
x_min, x_max = coords[1].min(), coords[1].max()
annotation["objects"].append({
"object_id": int(obj_id),
"bbox": [int(x_min), int(y_min), int(x_max), int(y_max)],
"area": int(mask.sum())
})
dataset_info["images"].append({
"id": i,
"filename": image_filename,
"width": 1024,
"height": 768
})
dataset_info["annotations"].append(annotation)
# Save individual annotation
with open(f"dataset/annotations/annotation_{i:06d}.json", 'w') as f:
json.dump(annotation, f, indent=2)
if i % 10 == 0:
print(f"Created {i+1}/100 samples")
# Save dataset metadata
with open("dataset/dataset_info.json", 'w') as f:
json.dump(dataset_info, f, indent=2)
print("Dataset creation complete!")
create_ml_dataset()Best Practices
1. Error Handling
Always handle potential errors gracefully:
from uesynth import UESynthClient, UESynthError
def robust_capture():
try:
with UESynthClient() as client:
# Test connection
if not client.is_connected():
raise ConnectionError("Failed to connect to UESynth")
# Your capture code here
image = client.capture.rgb()
except UESynthError as e:
print(f"UESynth specific error: {e}")
except ConnectionError as e:
print(f"Connection error: {e}")
except Exception as e:
print(f"Unexpected error: {e}")
robust_capture()2. Resource Management
Use context managers for automatic cleanup:
# ✅ Good - automatic cleanup
with UESynthClient() as client:
image = client.capture.rgb()
# Client automatically disconnected
# ❌ Manual cleanup required
client = UESynthClient()
try:
image = client.capture.rgb()
finally:
client.disconnect() # Must remember to disconnect3. Progress Tracking
For long-running operations, show progress:
from tqdm import tqdm
from uesynth import UESynthClient
def capture_with_progress():
with UESynthClient() as client:
positions = [(i*10, 0, 50) for i in range(100)]
for i, (x, y, z) in enumerate(tqdm(positions, desc="Capturing")):
client.camera.set_location(x=x, y=y, z=z)
image = client.capture.rgb()
cv2.imwrite(f"frame_{i:03d}.png", image)
capture_with_progress()4. Configuration Validation
Validate your setup before starting:
from uesynth import UESynthClient
def validate_setup():
try:
with UESynthClient() as client:
# Test basic functionality
location = client.camera.get_location()
print(f"✅ Camera accessible at: {location}")
# Test capture
test_image = client.capture.rgb(width=256, height=256)
print(f"✅ Capture working: {test_image.shape}")
# Test object access
objects = client.objects.list_all()
print(f"✅ Found {len(objects)} objects in scene")
return True
except Exception as e:
print(f"❌ Setup validation failed: {e}")
return False
if validate_setup():
print("Setup validation passed - ready to proceed!")
else:
print("Please fix setup issues before proceeding")This guide covers the fundamental usage patterns for UESynth. As you become more comfortable with these basics, you can explore advanced features like async streaming, scene randomization, and performance optimization.
Next Steps
Learn about Advanced Features
Explore Scene Randomization
Check out Data Collection Workflows
Last updated