Robot FPV Radio Controller
I created this code first to control a Spot doggy not by WASD but by my FPV Flysky i6 radio transmitter.
Installation
Fire up:
IsaacSim -> Window -> Examples -> Robotics Examples -> Policy -> Qudrupped -> Click 'Open Source Code' -> [it will fire up VSCode] -> paste the code below commenting the initial one
My code just extends the control, if no transmitter converted, it’ll set back to initial Arrow+N/M control over Spot. Written using pygame library, so it should be installed in Python version linked to your IsaacSim installation
For me
python3.10 -m pip install pygame
worked well, but check that you pip install tthings for the right python
In my setup the transmitter is connected via Trainer-USB wire to my laptop and is perceived as a PPM joystick device by Windows.
Some additional setups might be needed for other transmitters or older Windows versions. (Setup work fine on Ubuntu 22.04 also)
On Windows check the connection of your transmitter by using Win+R -> joy.cpl -> [see your PPM device] -> Properties In Properties you can see how your sticks are set. Use smth like vJoy or analogues to set up your sticks
vJoy seems to be compromised, so stick to analogues, I use joyToKey instead.
The plan is to publish extension for a radio control of any movable agent in Isaac by any FPV Controller.
Code
Здесь код для управления собакой Spot.
import carb
import numpy as np
import omni
import pygame
from isaacsim.examples.interactive.base_sample import BaseSample
from isaacsim.robot.policy.examples.robots import SpotFlatTerrainPolicy
class QuadrupedExample(BaseSample):
def __init__(self) -> None:
super().__init__()
# World settings
pygame.init()
pygame.display.init() # Explicitly initialize display (needed for event pump)
self._world_settings = {
"stage_units_in_meters": 1.0,
"physics_dt": 1.0 / 500.0,
"rendering_dt": 10.0 / 500.0
}
self._base_command = [0.0, 0.0, 0.0]
# Pygame initialization
pygame.init()
pygame.joystick.init()
# Controller state
self.joystick = None
self.controller_connected = False
self.last_axis_values = None
self.threshold = 0.1 # 10% deadzone
self.scale_factor = 2.0
self.setup_controller()
# Claude suggestion: Add setup for simulation callbacks
self.setup_simulation_callbacks()
# Fallback keyboard controls
self._input_keyboard_mapping = {
"UP": [2.0, 0.0, 0.0],
"DOWN": [-2.0, 0.0, 0.0],
"RIGHT": [0.0, -2.0, 0.0],
"LEFT": [0.0, 2.0, 0.0],
"N": [0.0, 0.0, 2.0],
"M": [0.0, 0.0, -2.0],
}
def setup_scene(self) -> None:
self._world.scene.add_default_ground_plane(
z_position=0,
name="default_ground_plane",
prim_path="/World/defaultGroundPlane",
static_friction=0.2,
dynamic_friction=0.2,
restitution=0.01,
)
self.spot = SpotFlatTerrainPolicy(
prim_path="/World/Spot",
name="Spot",
position=np.array([0, 0, 0.8]),
)
def setup_controller(self):
"""Initialize Pygame joystick connection"""
if pygame.joystick.get_count() > 0:
self.joystick = pygame.joystick.Joystick(0)
self.joystick.init()
self.controller_connected = True
self.last_axis_values = [0.0] * self.joystick.get_numaxes()
carb.log_info(f"Controller connected: {self.joystick.get_name()}")
carb.log_info(f"Axes: {self.joystick.get_numaxes()}, Buttons: {self.joystick.get_numbuttons()}")
else:
carb.log_warn("No controller found - using keyboard fallback")
self._setup_keyboard_fallback()
# Claude suggestion: Add method to setup simulation callbacks
def setup_simulation_callbacks(self):
"""Set up callbacks for simulation events"""
self._timeline = omni.timeline.get_timeline_interface()
self._stop_sub = self._timeline.get_timeline_event_stream().create_subscription_to_pop(
self._on_timeline_event
)
# Claude suggestion: Modified timeline event handler with world existence check
def _on_timeline_event(self, event):
"""Handle timeline events like stop/play"""
if event.type == int(omni.timeline.TimelineEventType.STOP):
carb.log_info("Simulation stopped, preparing for reset")
self._physics_ready = False
elif event.type == int(omni.timeline.TimelineEventType.PLAY):
carb.log_info("Simulation started, resetting state")
try:
self.reset()
except Exception as e:
carb.log_error(f"Error during reset: {str(e)}")
# Set flag to ensure initialization happens later
self._physics_ready = False
def _setup_keyboard_fallback(self):
"""Setup keyboard controls as fallback"""
self._appwindow = omni.appwindow.get_default_app_window()
self._input = carb.input.acquire_input_interface()
self._keyboard = self._appwindow.get_keyboard()
self._sub_keyboard = self._input.subscribe_to_keyboard_events(
self._keyboard, self._sub_keyboard_event)
# Claude suggestion: Add method to handle world initialization
def ensure_world_initialized(self):
"""Make sure world reference is initialized"""
if not hasattr(self, '_world') or self._world is None:
carb.log_info("Initializing world reference")
try:
self._world = self.get_world()
if self._world is not None:
# Set up physics callback
if not self._world.physics_callback_exists("physics_step"):
self._world.add_physics_callback("physics_step", callback_fn=self.on_physics_step)
return True
else:
carb.log_warn("World still not available")
return False
except Exception as e:
carb.log_error(f"Error initializing world: {str(e)}")
return False
return True
# Claude suggestion: Modified to ensure world is initialized
async def setup_post_load(self) -> None:
self._physics_ready = False
self.ensure_world_initialized()
try:
await self.get_world().play_async()
except Exception as e:
carb.log_error(f"Error in setup_post_load: {str(e)}")
# Claude suggestion: Modified to add additional checks and error handling
def on_physics_step(self, step_size) -> None:
# Ensure world is properly initialized
if not self.ensure_world_initialized():
return
# Process controller input
pygame.event.pump()
if self.controller_connected:
self._process_controller_input()
# Make sure spot robot is initialized
if not hasattr(self, 'spot') or self.spot is None:
carb.log_warn("Spot robot not initialized yet")
self._physics_ready = False
return
# Update robot physics - with additional safety checks
try:
if not self._physics_ready:
# Initialize or reinitialize on first step or after reload
self._physics_ready = True
self.spot.initialize()
self.spot.post_reset()
self.spot.robot.set_joints_default_state(self.spot.default_pos)
carb.log_info("Robot initialized successfully")
else:
# Check if articulation view exists before applying commands
if hasattr(self.spot.robot, '_articulation_controller') and \
hasattr(self.spot.robot._articulation_controller, '_articulation_view') and \
self.spot.robot._articulation_controller._articulation_view is not None:
self.spot.forward(step_size, self._base_command)
else:
# Physics components not ready yet, try to reinitialize
carb.log_warn("Physics components not ready, reinitializing...")
self._physics_ready = False
except Exception as e:
# Reset physics ready flag to force reinitialization on next step
self._physics_ready = False
carb.log_error(f"Error in physics step: {str(e)}")
def _process_controller_input(self):
"""Convert joystick input to robot commands"""
new_command = [0.0, 0.0, 0.0]
# Get current axis values (Xbox/PS4 mapping)
axes = [self.joystick.get_axis(i) for i in range(self.joystick.get_numaxes())]
print(axes)
# # Left Stick: Movement (Axis 0: LR, Axis 1: UD)
# roll = axes[0] if len(axes) > 0 else 0.0
# pitch = -axes[1] if len(axes) > 1 else 0.0 # Invert for natural control
# # Right Stick: Rotation (Axis 2: LR)
# yaw = axes[2] if len(axes) > 2 else 0.0
# WAITASEC!!!
# Right Stick: Movement (Axis 0: LR, Axis 1: UD)
roll = -axes[0] if len(axes) > 0 else 0.0
pitch = axes[1] if len(axes) > 1 else 0.0 # Invert for natural control
# Left Stick: Rotation (Axis 2: LR)
yaw = -axes[4] if len(axes) > 4 else 0.0 # Claude suggestion: Fixed index check from 2 to 4
# Apply deadzone and scaling
if abs(roll) > self.threshold:
new_command[1] = roll * self.scale_factor # Left/Right
if abs(pitch) > self.threshold:
new_command[0] = pitch * self.scale_factor # Forward/Back
if abs(yaw) > self.threshold:
new_command[2] = yaw * self.scale_factor # Rotation
# Print diagnostics when values change significantly
if self.last_axis_values:
changes = [abs(a - b) > 0.05 for a, b in zip(axes, self.last_axis_values)]
if any(changes):
carb.log_info(f"Controller Input - Roll: {roll:.2f}, Pitch: {pitch:.2f}, Yaw: {yaw:.2f}")
self._base_command = new_command
self.last_axis_values = axes
def _sub_keyboard_event(self, event, *args, **kwargs) -> bool:
"""Handle keyboard fallback controls"""
if event.type == carb.input.KeyboardEventType.KEY_PRESS:
if event.input.name in self._input_keyboard_mapping:
self._base_command += np.array(self._input_keyboard_mapping[event.input.name])
elif event.type == carb.input.KeyboardEventType.KEY_RELEASE:
if event.input.name in self._input_keyboard_mapping:
self._base_command -= np.array(self._input_keyboard_mapping[event.input.name])
return True
# Claude suggestion: Add safety checks for None world
def reset(self):
"""Reset the simulation state"""
self._physics_ready = False
self._base_command = [0.0, 0.0, 0.0]
# Reset controller state
if self.controller_connected:
self.last_axis_values = [0.0] * self.joystick.get_numaxes()
# Check if world exists before trying to access it
if self.ensure_world_initialized():
# Make sure any old physics callbacks are cleaned up
if self._world.physics_callback_exists("physics_step"):
self._world.remove_physics_callback("physics_step")
# Re-add the physics callback
self._world.add_physics_callback("physics_step", callback_fn=self.on_physics_step)
else:
carb.log_warn("World not available during reset, will initialize when ready")
def world_cleanup(self):
"""Clean up resources"""
if self.controller_connected:
pygame.quit()
if hasattr(self, '_sub_keyboard'):
self._sub_keyboard = None
if hasattr(self, '_world') and self._world is not None:
if self._world.physics_callback_exists("physics_step"):
self._world.remove_physics_callback("physics_step")