"use client"; import { useState, useEffect, useMemo, useRef, useCallback, startTransition, } from "react"; import { Card } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; import { Gamepad2, Keyboard, Settings, Square, Disc as Record, Check, } from "lucide-react"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; import { teleoperate, type TeleoperationProcess, type TeleoperationState, type TeleoperateConfig, type RobotConnection, } from "@lerobot/web"; import { getUnifiedRobotData } from "@/lib/unified-storage"; import { MemoRecorder as Recorder } from "@/components/recorder"; const UI_UPDATE_INTERVAL_MS = 100; interface RecordingViewProps { robot: RobotConnection; } export function RecordingView({ robot }: RecordingViewProps) { const [teleopState, setTeleopState] = useState({ isActive: false, motorConfigs: [], lastUpdate: 0, keyStates: {}, }); const [isInitialized, setIsInitialized] = useState(false); const [controlEnabled, setControlEnabled] = useState(false); const [selectedTeleoperatorType, setSelectedTeleoperatorType] = useState< "direct" | "keyboard" >("keyboard"); const [showConfigure, setShowConfigure] = useState(false); const [recorderCallbacks, setRecorderCallbacks] = useState<{ startRecording: () => Promise; stopRecording: () => Promise; isRecording: boolean; } | null>(null); // Current step tracking const currentStep = useMemo(() => { if (!robot.isConnected) return 0; // No steps active if robot offline if (!controlEnabled) return 1; // Step 1: Enable Control if (!recorderCallbacks?.isRecording) return 2; // Step 2: Start Recording return 3; // Step 3: Move Robot (recording active) }, [robot.isConnected, controlEnabled, recorderCallbacks?.isRecording]); const keyboardProcessRef = useRef(null); const directProcessRef = useRef(null); const lastUiUpdateRef = useRef(0); // Load calibration data from unified storage const calibrationData = useMemo(() => { if (!robot.serialNumber) return undefined; const data = getUnifiedRobotData(robot.serialNumber); if (data?.calibration) { return data.calibration; } // Return undefined if no calibration data - let library handle defaults return undefined; }, [robot.serialNumber]); // Initialize teleoperation for recording const initializeTeleoperation = useCallback(async () => { if (!robot || !robot.robotType) { return false; } try { // Create keyboard teleoperation process const keyboardConfig: TeleoperateConfig = { robot: robot, teleop: { type: "keyboard", }, calibrationData, onStateUpdate: (state: TeleoperationState) => { const now = performance.now(); if (now - lastUiUpdateRef.current >= UI_UPDATE_INTERVAL_MS) { lastUiUpdateRef.current = now; startTransition(() => setTeleopState(state)); } }, }; const keyboardProcess = await teleoperate(keyboardConfig); // Create direct teleoperation process const directConfig: TeleoperateConfig = { robot: robot, teleop: { type: "direct", }, calibrationData, onStateUpdate: (state: TeleoperationState) => { const now = performance.now(); if (now - lastUiUpdateRef.current >= UI_UPDATE_INTERVAL_MS) { lastUiUpdateRef.current = now; startTransition(() => setTeleopState(state)); } }, }; const directProcess = await teleoperate(directConfig); keyboardProcessRef.current = keyboardProcess; directProcessRef.current = directProcess; setTeleopState(directProcess.getState()); setIsInitialized(true); return true; } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to initialize teleoperation for recording"; toast({ title: "Teleoperation Error", description: errorMessage, variant: "destructive", }); return false; } }, [robot, robot.robotType, calibrationData]); // Enable robot control for recording const enableControl = useCallback(async () => { if (!robot.isConnected) { return false; } const success = await initializeTeleoperation(); if (success) { // Start the appropriate teleoperator based on selection if ( selectedTeleoperatorType === "keyboard" && keyboardProcessRef.current ) { keyboardProcessRef.current.start(); } else if ( selectedTeleoperatorType === "direct" && directProcessRef.current ) { directProcessRef.current.start(); } setControlEnabled(true); return true; } return false; }, [robot.isConnected, initializeTeleoperation, selectedTeleoperatorType]); // Disable robot control const disableControl = useCallback(async () => { if (keyboardProcessRef.current) { keyboardProcessRef.current.stop(); } if (directProcessRef.current) { directProcessRef.current.stop(); } setControlEnabled(false); }, [selectedTeleoperatorType]); // Keyboard event handlers (guarded to not interfere with inputs/shortcuts) const handleKeyDown = useCallback( (event: KeyboardEvent) => { if (!teleopState.isActive || !keyboardProcessRef.current) return; // Ignore when user is typing in inputs/textareas or contenteditable elements const target = event.target as HTMLElement | null; const isEditableTarget = !!( target && (target.tagName === "INPUT" || target.tagName === "TEXTAREA" || target.tagName === "SELECT" || target.isContentEditable || target.closest( '[role="textbox"], [contenteditable="true"], input, textarea, select' )) ); if (isEditableTarget) return; // Allow browser/system shortcuts (e.g. Ctrl/Cmd+R, Ctrl/Cmd+L, etc.) if (event.metaKey || event.ctrlKey || event.altKey) return; // Only handle specific teleop keys const rawKey = event.key; const normalizedKey = rawKey.length === 1 ? rawKey.toLowerCase() : rawKey; const allowedKeys = new Set([ "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "w", "a", "s", "d", "q", "e", "o", "c", "Escape", ]); if (!allowedKeys.has(normalizedKey)) return; event.preventDefault(); const keyboardTeleoperator = keyboardProcessRef.current.teleoperator; if (keyboardTeleoperator && "updateKeyState" in keyboardTeleoperator) { ( keyboardTeleoperator as { updateKeyState: (key: string, pressed: boolean) => void; } ).updateKeyState(normalizedKey, true); } }, [teleopState.isActive] ); const handleKeyUp = useCallback( (event: KeyboardEvent) => { if (!teleopState.isActive || !keyboardProcessRef.current) return; // Ignore when user is typing in inputs/textareas or contenteditable elements const target = event.target as HTMLElement | null; const isEditableTarget = !!( target && (target.tagName === "INPUT" || target.tagName === "TEXTAREA" || target.tagName === "SELECT" || target.isContentEditable || target.closest( '[role="textbox"], [contenteditable="true"], input, textarea, select' )) ); if (isEditableTarget) return; // Allow browser/system shortcuts (e.g. Ctrl/Cmd+R, Ctrl/Cmd+L, etc.) if (event.metaKey || event.ctrlKey || event.altKey) return; // Only handle specific teleop keys const rawKey = event.key; const normalizedKey = rawKey.length === 1 ? rawKey.toLowerCase() : rawKey; const allowedKeys = new Set([ "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "w", "a", "s", "d", "q", "e", "o", "c", "Escape", ]); if (!allowedKeys.has(normalizedKey)) return; event.preventDefault(); const keyboardTeleoperator = keyboardProcessRef.current.teleoperator; if (keyboardTeleoperator && "updateKeyState" in keyboardTeleoperator) { ( keyboardTeleoperator as { updateKeyState: (key: string, pressed: boolean) => void; } ).updateKeyState(normalizedKey, false); } }, [teleopState.isActive] ); // Register keyboard events useEffect(() => { if (teleopState.isActive && selectedTeleoperatorType === "keyboard") { window.addEventListener("keydown", handleKeyDown); window.addEventListener("keyup", handleKeyUp); return () => { window.removeEventListener("keydown", handleKeyDown); window.removeEventListener("keyup", handleKeyUp); }; } }, [ teleopState.isActive, selectedTeleoperatorType, handleKeyDown, handleKeyUp, ]); // Cleanup on unmount useEffect(() => { return () => { const cleanup = async () => { try { if (keyboardProcessRef.current) { await keyboardProcessRef.current.disconnect(); keyboardProcessRef.current = null; } if (directProcessRef.current) { await directProcessRef.current.disconnect(); directProcessRef.current = null; } } catch (error) { console.warn("Error during teleoperation cleanup:", error); } }; cleanup(); }; }, []); // Memoize teleoperators array for the Recorder component const memoizedTeleoperators = useMemo(() => { if (!controlEnabled) return []; // Return the active teleoperator based on selection const activeTeleoperator = selectedTeleoperatorType === "keyboard" ? keyboardProcessRef.current?.teleoperator : directProcessRef.current?.teleoperator; return activeTeleoperator ? [activeTeleoperator] : []; }, [controlEnabled, selectedTeleoperatorType]); return (
{/* Robot Movement Recorder Header */}

robot movement recorder

dataset{" "} recording{" "} interface

robot: {robot.isConnected ? "ONLINE" : "OFFLINE"}
{/* Step-by-Step Recording Guide */}

How to Record Robot Data

{/* Step 1 */}
1 ? "bg-green-500/20 border-green-500/50 text-green-400" // Completed : currentStep === 1 ? "bg-yellow-500/20 border-yellow-500/50 text-yellow-400" // Active : "bg-muted/20 border-muted/50 text-muted-foreground" // Inactive )} > {currentStep > 1 ? : "1"}
Enable Robot Control

Choose how you want to control the robot during recording.

{!controlEnabled && robot.isConnected && (

{selectedTeleoperatorType === "direct" ? "Programmatic control (use teleoperation page for manual sliders)" : "Move robot using keyboard keys (WASD, arrows, Q/E, O/C)"}

)}
{/* Step 2 */}
2 ? "bg-green-500/20 border-green-500/50 text-green-400" // Completed : currentStep === 2 ? "bg-yellow-500/20 border-yellow-500/50 text-yellow-400" // Active : "bg-muted/20 border-muted/50 text-muted-foreground" // Inactive )} > {currentStep > 2 ? : "2"}
Start Recording

Click "Start Recording" to begin capturing robot movements.

{null}
{/* Step 3 */}
3
Move the Robot

{currentStep === 3 ? ( 🎯 Recording active! Move the robot to demonstrate your task. ) : ( "Move the robot manually to demonstrate the task you want to teach. Recording will capture your movements." )}

{/* Robot Movement Recorder */}
); }