{ "at": "2026-10-05T18:02:45.547675+00:00", "scope": "Current catalog and latest evidence. Prior review is retained as a historical provenance file.", "selected_tasks": 32, "unresolved_selection_decisions": [ "robotwin/robotwin-aloha.place_dual_shoes" ], "pending_new_evidence": [], "remove_tasks": [], "decisions": { "Retain provisionally": 28, "Retain with stated limits": 3, "Review before final inclusion": 1 }, "rows": [ { "family": "task01", "slot": 1, "native_task_id": "robocasa/LoadCondimentsInFridge", "catalog_path": "tasks/task01/catalog/01-load-condiments-in-fridge/task.yaml", "catalog_sha256": "2aa9dd2770303990731a430321bda10bf8317f5647c0a70648d1a95692278d91", "current_instruction": "Place the condiments from the counter on the top fridge shelf. Move any existing top-shelf items to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "measured_instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-01-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-01-instruction02", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "Both condiments on the top rack; existing top-rack items on other racks; gripper clear of all four objects.", "finding": "Retain the dynamic food identities and release/retreat clause. The terminal floor placements miss the storage goal itself.", "structural_difficulty": "Shelf rearrangement and collision-aware transport.", "learning_hypothesis": "Reusable routines for shelf rearrangement and collision-aware transport. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/loading_fridge/load_condiments_in_fridge.py", "sha256": "374a8bf1a2a0b7ec8eb6df4c471b162d9c8e156d35d28580afdb6cc574807b3d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 155.9, "step_limit": 20000, "steps": 3118, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 164, "model_tool_calls": 163, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "LoadCondimentsInFridge" }, "backend_task": "robocasa.LoadCondimentsInFridge", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/LoadCondimentsInFridge", "instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-01-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 2, "native_task_id": "robocasa/FilterMicrowavableItem", "catalog_path": "tasks/task01/catalog/02-filter-microwavable-item/task.yaml", "catalog_sha256": "8ca8d7973c981faece4daf827ab92a4380b60c8d127523b4036192376e4f796c", "current_instruction": "Move the non-microwavable food named in the scene instruction from the bowl onto the small plate. Put the bowl containing only the microwavable food in the microwave, close the door, and press start. Release the bowl and withdraw the gripper well clear of it.", "measured_instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-02-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-02-instruction02", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Fruit on the plate, meat in the bowl, bowl in the microwave, door closed, microwave on, gripper clear of bowl.", "finding": "The sampled native labels identify the non-microwavable fruit and microwavable meat. Terminal evidence does not show the required bowl-and-meat containment.", "structural_difficulty": "Selective removal, nested containment, door and switch control.", "learning_hypothesis": "Reusable routines for selective removal, nested containment, door and switch control. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/microwaving_food/filter_microwavable_item.py", "sha256": "89ef510ac1ad0e1fd13094e477a1bed00a2361a351625778fe8cd8289aeab594", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 297.4, "step_limit": 20000, "steps": 5948, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 283, "model_tool_calls": 282, "measured_protocol": { "episode_specs": [ { "instruction_override": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "FilterMicrowavableItem" }, "backend_task": "robocasa.FilterMicrowavableItem", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/FilterMicrowavableItem", "instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-02-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 3, "native_task_id": "robocasa/StoreDumplings", "catalog_path": "tasks/task01/catalog/03-store-dumplings/task.yaml", "catalog_sha256": "2bd709f6e17025d223adf615c11d35cd4b7d40f5d1251297de81d469c947f3ae", "current_instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "measured_instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task01-03-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-03-instruction02-providerrecovery01", "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "Exactly two dumplings per container, both containers on fridge racks, gripper clear of food and containers.", "finding": "Keep side-by-side placement: the native receptacle helper requires direct contact, so stacking a dumpling on another can miss it. Containers remain outside the fridge in terminal evidence.", "structural_difficulty": "Counting, packing, and transporting filled containers.", "learning_hypothesis": "Reusable routines for counting, packing, and transporting filled containers. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/storing_leftovers/store_dumplings.py", "sha256": "666a531feb0a6a250953ccb0eebf32b72d763660aecce198b504dc722593ae77", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 830.75, "step_limit": 20000, "steps": 16615, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 142, "model_tool_calls": 141, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "StoreDumplings" }, "backend_task": "robocasa.StoreDumplings", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/StoreDumplings", "instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-03-instruction02-providerrecovery01/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 4, "native_task_id": "robocasa/DivideBuffetTrays", "catalog_path": "tasks/task01/catalog/04-divide-buffet-trays/task.yaml", "catalog_sha256": "ceea57e5a9b3fc54ec2aae8c39284dc20b809b1ae592219956ec619dd8672571", "current_instruction": "Place the two food groups named in the scene instruction on separate dining-counter trays, one group per tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "measured_instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-04-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-04-instruction02-providerrecovery01", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Two sampled food groups on separate trays with no cross-group membership; gripper clear of the food and trays.", "finding": "Group names must remain episode-specific. Food remains in the fridge; the failed result is not just a missing retreat. The initial camera does not show the whole workspace.", "structural_difficulty": "Category allocation and multi-location transport.", "learning_hypothesis": "Reusable routines for category allocation and multi-location transport. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/arranging_buffet/divide_buffet_trays.py", "sha256": "ecd23b0480a93254ea14f70d3141d450479f83df5c093f41a09cdcad90f1b545", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 783.65, "step_limit": 20000, "steps": 15673, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 210, "model_tool_calls": 209, "measured_protocol": { "episode_specs": [ { "instruction_override": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "DivideBuffetTrays" }, "backend_task": "robocasa.DivideBuffetTrays", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/DivideBuffetTrays", "instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-04-instruction02-providerrecovery01/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 5, "native_task_id": "robocasa/MakeCheesecakeFilling", "catalog_path": "tasks/task01/catalog/05-make-cheesecake-filling/task.yaml", "catalog_sha256": "e8c06e4f099765d6b9d30641623d5105296398df24184731195bd964238a0ce1", "current_instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "measured_instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task01-05-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-05-instruction02", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Butter, sugar cube and cream cheese in mixer bowl; mixer head lowered; speed dial on.", "finding": "The lowered-head clause is necessary and already concise. The recorded speed dial remains off. No requirement to produce a visibly mixed substance is added.", "structural_difficulty": "Ingredient insertion and sequential appliance operation.", "learning_hypothesis": "Reusable routines for ingredient insertion and sequential appliance operation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/mixing_ingredients/make_cheesecake_filling.py", "sha256": "905f1185f5f72e42bb14ed4e8f8afa282859d2b093e704743c2ff83cdf66b7ad", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 730.05, "step_limit": 20000, "steps": 14601, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 282, "model_tool_calls": 281, "measured_protocol": { "episode_specs": [ { "instruction_override": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "MakeCheesecakeFilling" }, "backend_task": "robocasa.MakeCheesecakeFilling", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/MakeCheesecakeFilling", "instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-05-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 6, "native_task_id": "robocasa/MultistepSteaming", "catalog_path": "tasks/task01/catalog/06-multistep-steaming/task.yaml", "catalog_sha256": "597068f649a9f1104372843ffd5262c03203afadca9e818208db6f37df505851", "current_instruction": "Turn on the sink faucet and rinse the food item named in the scene instruction in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the specified burner.", "measured_instruction": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-06-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-06-completion01", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Faucet turned on; sampled food in sink while water runs; faucet turned off; same food in pot; pot on sampled burner.", "finding": "Keep 'that same item': the native wording can call a sampled lemon wedge a vegetable. The source does not require turning on the stove or waiting for actual steaming.", "structural_difficulty": "Ordered state changes and transfer of a wet item and pot.", "learning_hypothesis": "Reusable routines for ordered state changes and transfer of a wet item and pot. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/steaming_food/multistep_steaming.py", "sha256": "e745045ff2b19773ca6ff33fd3485daad38266bb772d06ea5f112c3ad8eff94d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 464.4, "step_limit": 20000, "steps": 9288, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 360, "model_tool_calls": 359, "measured_protocol": { "episode_specs": [ { "instruction_override": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "MultistepSteaming" }, "backend_task": "robocasa.MultistepSteaming", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/MultistepSteaming", "instruction": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-06-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20.0, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 7, "native_task_id": "robocasa/ScalePortioning", "catalog_path": "tasks/task01/catalog/07-scale-portioning/task.yaml", "catalog_sha256": "9be088861c6eed90e19076ff1c73e78424c66c89827c129e64d4d29d570890ce", "current_instruction": "Place the specified food from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "measured_instruction": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-07-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-07-completion01", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Food on the scale while gripper is clear for the native reading interval, then food on the dining-counter plate with gripper clear.", "finding": "Retain release, waiting, and final retreat. The native weighing state is latched. The terminal food did not reach the complete weigh-and-plate goal.", "structural_difficulty": "Temporary placement, dwell time, and final relocation.", "learning_hypothesis": "Reusable routines for temporary placement, dwell time, and final relocation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/portioning_meals/scale_portioning.py", "sha256": "50b6fa4bd2415d18ad6cb3e4f9ac7021cfdc47d8720b5700c1a4e97156708579", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 243.9, "step_limit": 20000, "steps": 4878, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 213, "model_tool_calls": 212, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "ScalePortioning" }, "backend_task": "robocasa.ScalePortioning", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/ScalePortioning", "instruction": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-07-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20.0, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 8, "native_task_id": "robocasa/ScrubCuttingBoard", "catalog_path": "tasks/task01/catalog/08-scrub-cutting-board/task.yaml", "catalog_sha256": "8fa3be64f47bcd0f4bbe12a43a02cddcfef25ce5cc72820c53f192ab564ec378", "current_instruction": "Pick up the sponge and scrub across different parts of the cutting board with several broad sweeps, keeping the sponge pressed against the board. Put down and release the sponge, then withdraw the gripper well clear of it.", "measured_instruction": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task01-08-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-08-instruction02", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "Held sponge contacts the board at multiple separated positions, covers the required extent, then gripper moves clear.", "finding": "Several broad contact sweeps retain the native coverage condition; a single press does not. Remove the repeated explanation of broad coverage. Native success is geometric contact coverage, not simulated dirt removal.", "structural_difficulty": "Tool contact, surface coverage, and release.", "learning_hypothesis": "Reusable routines for tool contact, surface coverage, and release. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/sanitizing_cutting_board/scrub_cutting_board.py", "sha256": "3a4082816ead472ea533bad42fe11a198c8baf10ded05abeef0fbb58e1b1e3e0", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 75.2, "step_limit": 20000, "steps": 1504, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 46, "model_tool_calls": 45, "measured_protocol": { "episode_specs": [ { "instruction_override": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "ScrubCuttingBoard" }, "backend_task": "robocasa.ScrubCuttingBoard", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/ScrubCuttingBoard", "instruction": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-08-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task02", "slot": 1, "native_task_id": "libero/10_9", "catalog_path": "tasks/task02/catalog/01-mug-in-microwave/task.yaml", "catalog_sha256": "2e4f38c3327a60978fdc1f9bd67a9d41edb7469d2cf3cae4431d6b86fae9ae5d", "current_instruction": "put the yellow and white mug in the microwave and close it", "measured_instruction": "put the yellow and white mug in the microwave and close it", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "t03-kinex-v0.8.0-episode_01", "measured_task_id": 3, "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "In(white_yellow_mug, microwave_heating_region) and Close(microwave).", "finding": "Native BDDL byte hash matches the measured run. The yellow-and-white mug is distinguishable from the other mug. Both original clauses map directly to the goal; keep unchanged.", "structural_difficulty": "Placement into an articulated fixture and door closure.", "learning_hypothesis": "Reusable routines for placement into an articulated fixture and door closure. No RSI gain is measured by this audit.", "source": { "relative_path": "bddl_files/libero_10/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it.bddl", "sha256": "456d145f92be049f445fc77673dc583d9d17ea7afe92f6ffcb2fd1fa5565d420", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Retained pinned BDDL snapshot matching the measured initialization hash" }, "verdict": { "episode": 1, "initialization": { "bddl_sha256": "456d145f92be049f445fc77673dc583d9d17ea7afe92f6ffcb2fd1fa5565d420", "control_frequency_hz": 20, "controller": "JOINT_POSITION/set_qpos", "init_state_index": 0, "init_state_sha256": "6b75c8d4d1e1c40b5d99c78ceea380d99ccffb0a484a4cb2d3153488a0afbbe2", "initialization_frames": 5, "loaded_state_sha256": "ea7db2b727794199cfa1e42f8d924c5841e4af18716a392adf30d5517114c87f", "max_steps": 20000, "seed": 0, "suite": "libero_10", "task_id": 9, "task_order_index": 0 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 266.35, "step_limit": 20000, "steps": 5327, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 103, "model_tool_calls": 112, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "libero", "backend_kwargs": { "init_state_index": 0, "max_steps": 20000, "stop_on_success": true, "suite_name": "libero_10", "task_id": 9 }, "backend_task": "libero.10_9", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "libero/10_9", "instruction": "put the yellow and white mug in the microwave and close it", "robot": "libero/libero-panda-kitchen", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "libero-curated", "version": "0.7.0", "backend": "libero", "selection": "all", "protocol": { "episodes": 3, "seeds": [ 0, 1, 2 ], "initial_state_indices": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/t03/kinex-v0.8.0/episode_01/edited.mp4", "playback": { "command_resimulation": false, "fps": 20, "intermediate_states_verified": null, "kind": "recorded-native-integration-states", "source_fps": 20.0, "speed": 4, "validation_scope": null, "waiting_policy": "Synchronous model waiting omitted; recorded native states retained in HD sources" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task02", "slot": 2, "native_task_id": "libero/10_3", "catalog_path": "tasks/task02/catalog/02-bowl-in-drawer/task.yaml", "catalog_sha256": "529b914ad40d70ecfc9f62b59c4f0567a5c62e7d4a8617c2833b5524e0ea3bfe", "current_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "measured_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "t04-kinex-v0.8.0-episode_01", "measured_task_id": 4, "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "In(black_bowl, cabinet_bottom_region) and Close(cabinet_bottom_region).", "finding": "Native BDDL byte hash matches the measured run. The black bowl and bottom drawer are identifiable. Keep the original instruction unchanged.", "structural_difficulty": "Drawer access, containment, and closure.", "learning_hypothesis": "Reusable routines for drawer access, containment, and closure. No RSI gain is measured by this audit.", "source": { "relative_path": "bddl_files/libero_10/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it.bddl", "sha256": "5255fe54d7f25fad4dee8fa30a30033d8cb908f1708d953c91ab609264fb4fb8", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Retained pinned BDDL snapshot matching the measured initialization hash" }, "verdict": { "episode": 1, "initialization": { "bddl_sha256": "5255fe54d7f25fad4dee8fa30a30033d8cb908f1708d953c91ab609264fb4fb8", "control_frequency_hz": 20, "controller": "JOINT_POSITION/set_qpos", "init_state_index": 0, "init_state_sha256": "e98ded151f64a036c74c872336dd09b968c379f3c9db7bc333aba37e7c235b86", "initialization_frames": 5, "loaded_state_sha256": "ea2ad0a67ceabe915bc4744e8231bf7fe71bddb74ff8c67eae6c821ff82078a4", "max_steps": 20000, "seed": 0, "suite": "libero_10", "task_id": 3, "task_order_index": 0 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 794.1, "step_limit": 20000, "steps": 15882, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 200, "model_tool_calls": 204, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "libero", "backend_kwargs": { "init_state_index": 0, "max_steps": 20000, "stop_on_success": true, "suite_name": "libero_10", "task_id": 3 }, "backend_task": "libero.10_3", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "libero/10_3", "instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "robot": "libero/libero-panda-kitchen", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "libero-curated", "version": "0.7.0", "backend": "libero", "selection": "all", "protocol": { "episodes": 3, "seeds": [ 0, 1, 2 ], "initial_state_indices": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/t04/kinex-v0.8.0/episode_01/edited.mp4", "playback": { "command_resimulation": false, "fps": 20, "intermediate_states_verified": null, "kind": "recorded-native-integration-states", "source_fps": 20.0, "speed": 4, "validation_scope": null, "waiting_policy": "Synchronous model waiting omitted; recorded native states retained in HD sources" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 1, "native_task_id": "robotwin/robotwin-aloha.handover_block", "catalog_path": "tasks/task03/catalog/01-handover-block/task.yaml", "catalog_sha256": "28a8cf9167406c4d078635aec14a6bfb6ae76fe929e30a35eaf3d3cb69e30b7d", "current_instruction": "Lift the block with the left arm, pass it to the right arm, then place it upright on the blue pad.", "measured_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-01-kinex-v0.8.2-episode_01", "measured_task_id": "task03-01", "previous_slot": 1, "recommendation": "Retain with stated limits", "native_conditions": "Block functional point reaches the blue pad pose and the right gripper has an open command.", "finding": "Conditional retention only under the native final-state metric. The predicate has no history of left-to-right transfer; it cannot independently certify the requested handover. Do not weaken the instruction to turn this into plain placement. Gripper commands also do not prove completed physical release.", "structural_difficulty": "Bimanual transfer and upright placement, if the trajectory actually performs the transfer.", "learning_hypothesis": "Reusable routines for bimanual transfer and upright placement, if the trajectory actually performs the transfer. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/handover_block.py", "sha256": "5439712296d7e143d41f8e78b4ab86cd4fad7fb98f40f5f5abb067a462d17650", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 6023, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "cba5c9b6436448e0067283925eaaf21662dcfd9b046e79a5ab3df813763a1a17", "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 11625, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "task_config": "demo_clean", "wall_time_s": 43.96118344925344 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 69.840003317222, "step_limit": 20000, "steps": 1746, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 35, "model_tool_calls": 39, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "handover_block" }, "backend_task": "robotwin-aloha.handover_block", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.handover_block", "instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 2, "native_task_id": "robotwin/robotwin-aloha.place_dual_shoes", "catalog_path": "tasks/task03/catalog/02-place-dual-shoes/task.yaml", "catalog_sha256": "58a5a9d9f740fbf97607da25acc7150c8ff1f5224168ade864fe5eb1c1ed29b0", "current_instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes.", "measured_instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-02-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task03-02-instruction02", "previous_slot": 2, "recommendation": "Review before final inclusion", "native_conditions": "Both shoe poses near fixed left-front/right-back targets with toes left; both native gripper-open flags true.", "finding": "The native predicate DOES check both gripper-open flags, but those flags read commanded opening. In the measured journal, right-gripper opening is commanded at frame 1852 and success occurs at frame 1853 while the right finger joints remain about 0.0029 m, far below their approximately 0.045 m open pose. Physical release and stable hands-off placement are unproved. Recommend withholding this task from an unconditional final set under the unchanged predicate. It remains selected here because this request synchronizes the existing catalog rather than authorizing a new selection deletion.", "structural_difficulty": "Dual object reorientation and constrained placement.", "learning_hypothesis": "Reusable routines for dual object reorientation and constrained placement. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/place_dual_shoes.py", "sha256": "f35c4a133e1de567b274cbd15f158f585eb05e32aeb05be88839d249d0d7ab9e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 5883, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "a715656d4d3fff3a16238f57b8b2cbe6574b5fc164600c21f74bf7c3f8156513", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 11485, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 43.20029250346124 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 74.12000352051109, "step_limit": 20000, "steps": 1853, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 78, "model_tool_calls": 80, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "max_steps": 20000, "seed": 0, "task": { "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "place_dual_shoes" }, "backend_task": "robotwin-aloha.place_dual_shoes", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.place_dual_shoes", "instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-02-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 3, "native_task_id": "robotwin/robotwin-aloha.put_bottles_dustbin", "catalog_path": "tasks/task03/catalog/03-put-bottles-dustbin/task.yaml", "catalog_sha256": "1bba9b202b74698e9f71100d36492fcb0751852dd227665000ce14ae546b3edb", "current_instruction": "Move all three bottles from the table into the dustbin on the left.", "measured_instruction": "Move all three bottles from the table into the dustbin on the left. Finish with all three bottles inside the dustbin.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-05-kinex-v0.8.2-episode_01", "measured_task_id": "task03-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "All three bottle actor centers inside the fixed dustbin volume.", "finding": "The initial three bottles and left bin match the wording. Terminal views show bottles in the bin. Remove the repeated destination sentence. The native center test is not full-mesh containment, hands-off stability, or a handover test.", "structural_difficulty": "Repeated grasping and disposal of different bottle shapes.", "learning_hypothesis": "Reusable routines for repeated grasping and disposal of different bottle shapes. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/put_bottles_dustbin.py", "sha256": "1c03a4b227daa7d285fa366944f1f75773e32dc6a7b73ab693217a13fcd42795", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": "AssertionError: target_pose cannot be None for move action.", "expert_physics_steps": 16027, "expert_plan_success": true, "expert_prepass": true, "loaded_state_sha256": "1b1ea0b3953b47202b86cd6ac2ffc5cd2dac0d741937943832cae9e93f50a6e7", "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 22829, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "task_config": "demo_clean", "wall_time_s": 117.95161530468613 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 62.960002990439534, "step_limit": 20000, "steps": 1574, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 56, "model_tool_calls": 58, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "put_bottles_dustbin" }, "backend_task": "robotwin-aloha.put_bottles_dustbin", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.put_bottles_dustbin", "instruction": "Move all three bottles from the table into the dustbin on the left. Finish with all three bottles inside the dustbin.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted", "excluded_initialization_diagnostics": [ "expert_error", "expert_plan_success" ], "comparison_tolerance_unchanged": true }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 4, "native_task_id": "robotwin/robotwin-aloha.scan_object", "catalog_path": "tasks/task03/catalog/04-scan-object/task.yaml", "catalog_sha256": "2ac38bcf96e23a252a278cd9ec0499837bde2cf0fe5aad9312d8e821cf048e66", "current_instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "measured_instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task03-06-instruction03-kinex-v0.9.0-episode_01", "measured_task_id": "task03-06-instruction03", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Both grippers commanded closed; box bottom-center reference close to and in front of scanner functional point, with bounded lateral offset.", "finding": "The instruction identifies the two held objects, the scanner optical window, the center of the box bottom, opposing faces, and an approximate 2\u20133 cm separation. Ten static physical-surface checks across all five scanner models pass the exact native predicate. Under the stated selection criterion, correct execution of this clear goal can pass; the verifier need not reject every incomplete pose. The live checker can trigger before the faces become parallel, so this native success does not independently verify precise face orientation or the literal surface gap. The task retains grasp stability, object-frame reasoning, inverse-kinematics constraints and two-arm alignment. One successful seed establishes a successful native attempt under this protocol; it does not estimate general success rate, prove instruction changes caused the improvement, or establish an RSI learning gain. It should no longer be described as unsolved in this latest run.", "structural_difficulty": "Relative six-dimensional alignment while holding two objects.", "learning_hypothesis": "Reusable routines for relative six-dimensional alignment while holding two objects. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/scan_object.py", "sha256": "860dfe2cc86d180096089bc87502c7904f7ce2efb64fb32620c1720ca0178c2b", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 3070, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "6ce2d4764c6abe9d08b1eabef8ced34805aa4dc7bb02848bdbf86fca92112ed6", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 8672, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 71.6042634230107 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 162.4800077173859, "step_limit": 20000, "steps": 4062, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 57, "model_tool_calls": 56, "measured_protocol": { "episode_specs": [ { "instruction_override": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "max_steps": 20000, "seed": 0, "task": { "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "scan_object" }, "backend_task": "robotwin-aloha.scan_object", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.scan_object", "instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-06-instruction03/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false, "latest_evidence": "scanner-review.json", "static_positive_alignment_cases": 10 }, { "family": "task04", "slot": 1, "native_task_id": "robodojo/make_toast", "catalog_path": "tasks/task04/catalog/01-robodojo-make-toast/task.yaml", "catalog_sha256": "2385eb2103749e2dd1e940ab2fe7b26683abf3c3de6b7754818b12e3ff7c016b", "current_instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "measured_instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-01-instruction01-kinex-v0.8.2-episode_01", "measured_task_id": "task04-01-instruction01", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "One bread slice in each toaster slot, all slices upright, remaining slices in the rack, lever down, arms home.", "finding": "The four slices, two slots and rack are visible. Keep the count, one-per-slot, rack, lever and arm-return conditions. The task does not evaluate elapsed cooking or toast color.", "structural_difficulty": "Thin-object insertion, articulated actuation, and multi-stage cleanup.", "learning_hypothesis": "Reusable routines for thin-object insertion, articulated actuation, and multi-stage cleanup. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/make_toast.py", "sha256": "1047f484a73d1f92304393879b51f87e33d661b6ad7937198352fe37a0d3233d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "f58ca00a4fa671677d9292e7b843cd391932e6867abc2f33a8cd8df9a2e0e9e6", "wall_time_s": 66.90659895632416 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 432.04, "step_limit": 20000, "steps": 10801, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 328, "model_tool_calls": 353, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "make_toast" }, "backend_task": "robodojo.make_toast", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/make_toast", "instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-01-instruction01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 2, "native_task_id": "robodojo/classify_objects_by_language", "catalog_path": "tasks/task04/catalog/02-robodojo-classify-objects-by-language/task.yaml", "catalog_sha256": "13cbc16d3666cf3d0e7c4a52dda0e660b1ebe6867a676929a46f80b4df938440", "current_instruction": "Sort all objects into the baskets according to the current goal's category assignments. Place them fully inside, then return both arms to their starting poses.", "measured_instruction": "Put car objects into the left basket, watch objects into the middle basket, and wooden_toy objects into the right basket, then reset the robot arm.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task04-02-recovery01-kinex-v0.8.2-episode_01", "measured_task_id": "task04-02-recovery01", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Every object inside its episode-assigned basket, below its upper boundary, and both arms home.", "finding": "Keep the native reset-resolved category-to-basket mapping. A static generic override would erase the assignments. The catalog adds full insertion and arm return without fixing seed-zero categories.", "structural_difficulty": "Language-conditioned categorization and repeated sorting.", "learning_hypothesis": "Reusable routines for language-conditioned categorization and repeated sorting. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/classify_objects_by_language.py", "sha256": "242a8f048f487eb174ef246be78613bbb5f153549ee11a2650412718cf3b36c9", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "cedae218bff456f1033290de4bf91564e04911f4aec0cd573b70c6220a064517", "wall_time_s": 64.84979563206434 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 262.96, "step_limit": 20000, "steps": 6574, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 102, "model_tool_calls": 111, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "classify_objects_by_language" }, "backend_task": "robodojo.classify_objects_by_language", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/classify_objects_by_language", "instruction": "Put car objects into the left basket, watch objects into the middle basket, and wooden_toy objects into the right basket, then reset the robot arm.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-02-recovery01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 3, "native_task_id": "robodojo/store_laptop_and_headphones", "catalog_path": "tasks/task04/catalog/03-robodojo-store-laptop-and-headphones/task.yaml", "catalog_sha256": "7e044ee32208d5ae3e7440ad6894d775f6df0d05a1126ed7424ca6aa3a974ee3", "current_instruction": "Hang the headphones by the middle of their headband, aligned with the stand cradle so both earcups hang evenly below it. Close the laptop fully and seat it upright in its vertical stand. Release both objects and return both arms to their starting poses.", "measured_instruction": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-04-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-04-completion01", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Headband bridge aligned with stand cradle in position and orientation; laptop closed, upright and inserted in its stand; arms home.", "finding": "Keep earcup/headband orientation and fully closed, upright laptop seating. The latest headphone checks pass; laptop upright/stand checks fail. Wording can be shortened without deleting these goals.", "structural_difficulty": "Hanging a shaped object and articulated-object insertion.", "learning_hypothesis": "Reusable routines for hanging a shaped object and articulated-object insertion. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/store_laptop_and_headphones.py", "sha256": "2bee4e31f0594900629fe7f43266e16fecb310b152930ce51524507136809a92", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "d804229232414a2f4446497c48f57cc03621f2552f2c33ddf0705c5b2b3d39c5", "wall_time_s": 52.43044571578503 }, "native_reward": 0.2, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 781.36, "step_limit": 20000, "steps": 19534, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 324, "model_tool_calls": 323, "measured_protocol": { "episode_specs": [ { "instruction_override": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "store_laptop_and_headphones" }, "backend_task": "robodojo.store_laptop_and_headphones", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/store_laptop_and_headphones", "instruction": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-04-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 4, "native_task_id": "robodojo/cover_blocks", "catalog_path": "tasks/task04/catalog/04-robodojo-cover-blocks/task.yaml", "catalog_sha256": "9b3734d07c57b98f83b864350fcdf48b2f6ea317f2527f0d38169e9bec993ae4", "current_instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "measured_instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-05-kinex-v0.8.2-episode_01", "measured_task_id": "task04-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Cover left-to-right then uncover red-green-blue in sequence; cups inverted; blocks remain near initial locations; arms home at the end.", "finding": "Native reward checks six ordered stages. The wording already expresses the needed sequence and invariants; leave it unchanged. First and last views alone do not establish the history, so the sequential native verdict is essential.", "structural_difficulty": "Occlusion memory, ordered manipulation, and invariant preservation.", "learning_hypothesis": "Reusable routines for occlusion memory, ordered manipulation, and invariant preservation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/cover_blocks.py", "sha256": "9d14931b16842b93bea8abf9a07991a93a7e663b93c5fd1695108c8b922ab862", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "08430add27614badf9e3ddf928e22b147a66929e1ea16e717721c49aac7af5df", "wall_time_s": 62.40864769835025 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 196.68, "step_limit": 20000, "steps": 4917, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 68, "model_tool_calls": 73, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "cover_blocks" }, "backend_task": "robodojo.cover_blocks", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/cover_blocks", "instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 5, "native_task_id": "robodojo/play_Xylophone", "catalog_path": "tasks/task04/catalog/05-robodojo-play-xylophone/task.yaml", "catalog_sha256": "fb7f696453717db0c04189de680e456cb59d1cc7bd2073c594c7912f51c04f8c", "current_instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "measured_instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-06-kinex-v0.8.2-episode_01", "measured_task_id": "task04-06", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Mallet strike point reaches the eight key regions in order, with sufficient lift between successive strikes.", "finding": "Keep the original left-to-right strike instruction. No native arm-return requirement exists. Strike detection is geometric, not a force or audio-quality metric.", "structural_difficulty": "Tool use, repeated targeting, and ordered contact.", "learning_hypothesis": "Reusable routines for tool use, repeated targeting, and ordered contact. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/play_Xylophone.py", "sha256": "b7a4b0bc0457bcf55d714ca1867476fc4aabc53084adfc78d3ff5d1437bf7847", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "2c0b65e83b578e4be99357d487da0acecf0bb83c17a4767dacd3a17044313a4d", "wall_time_s": 47.931230374611914 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 120.04, "step_limit": 20000, "steps": 3001, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 49, "model_tool_calls": 48, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "play_Xylophone" }, "backend_task": "robodojo.play_Xylophone", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/play_Xylophone", "instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-06/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 4.166666666666667, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 6, "native_task_id": "robodojo/store_tools_in_toolbox", "catalog_path": "tasks/task04/catalog/06-robodojo-store-tools-in-toolbox/task.yaml", "catalog_sha256": "f020d7b09324f3f290f30e847333fcac2b8d50393625494c6a71ceabb2857901", "current_instruction": "Place each tool flat in its matching shaped recess, aligned with the outline and fully below the toolbox rim. Release all tools and return both arms to their starting poses.", "measured_instruction": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-07-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-07-completion01", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Each of four tools near and aligned with its matching recess and contained below the toolbox top; arms home.", "finding": "The shaped recesses identify destinations. Native helpers reject the latest slot alignments; hammer and pliers also miss full containment. Keep flat, matched outline, and below-rim seating while deleting repeated containment phrasing.", "structural_difficulty": "Shape matching, orientation control, and tight packing.", "learning_hypothesis": "Reusable routines for shape matching, orientation control, and tight packing. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/store_tools_in_toolbox.py", "sha256": "2969fc3eb171560b3c0bcea62b9f38b8b35f3aecc9bcc7085c3b1cec365b48fc", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "15029891c722a98c9f1b3c92d88dd5883af61f88169c15c35ff4194ea13e4527", "wall_time_s": 59.26869597937912 }, "native_reward": 0.5, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 773.6, "step_limit": 20000, "steps": 19340, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 153, "model_tool_calls": 152, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "store_tools_in_toolbox" }, "backend_task": "robodojo.store_tools_in_toolbox", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/store_tools_in_toolbox", "instruction": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-07-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 7, "native_task_id": "robodojo/insert_tubes", "catalog_path": "tasks/task04/catalog/07-robodojo-insert-tubes/task.yaml", "catalog_sha256": "a552f7fa5e490120a39ed373f0ea68b6e082f690d30929af1e707e77ffdc2912", "current_instruction": "Insert all three tubes upright and deeply into the rack, pointed ends down and capped ends up. Return both arms to their starting poses.", "measured_instruction": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-08-hard02-kinex-v0.9.0-episode_01", "measured_task_id": "task04-08-hard02", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "All three tubes upright, inside the rack, inserted more than 45 mm, and arms home.", "finding": "Pointed ends down and capped ends up match the saved-layout assets. Recorded insertion depths are about 54.75, 54.85 and 56.39 mm. Remove the duplicate seated-in-rack sentence while retaining deep insertion and arm return.", "structural_difficulty": "Reorientation and three constrained insertions.", "learning_hypothesis": "Reusable routines for reorientation and three constrained insertions. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/insert_tubes.py", "sha256": "9a68795ddbd0b816ffd7627e673390ecd83594c0e9d374d4e3abf3435ad439e1", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "7975670facd722b7366c55760a3bef7de735d9e203c6cc915f89108294f5269c", "wall_time_s": 54.14958868827671 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 204.68, "step_limit": 20000, "steps": 5117, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 86, "model_tool_calls": 85, "measured_protocol": { "episode_specs": [ { "instruction_override": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "insert_tubes" }, "backend_task": "robodojo.insert_tubes", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/insert_tubes", "instruction": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-08-hard02/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 8, "native_task_id": "robodojo/deposit_coin", "catalog_path": "tasks/task04/catalog/08-robodojo-deposit-coin/task.yaml", "catalog_sha256": "0fd54af41bcc4695b623c5219c1e7942abd08410f80b894cb8c61c97ef9e18b8", "current_instruction": "Pick up the coin from its holder and deposit it through the slot of the coin bank. Let the coin fall fully inside the bank, then return both arms to their starting poses.", "measured_instruction": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-09-candidate01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-09-candidate01", "previous_slot": 9, "recommendation": "Retain provisionally", "native_conditions": "Entire coin contained inside the bank's lower internal region and arms home.", "finding": "Keep the already revised instruction. It correctly requires full entry, without imposing bottom contact or resting. Native score is containment; it does not explicitly record the slot-crossing trajectory. The historical measured text differs from the current revised text.", "structural_difficulty": "Thin-object pickup, regrasping, and slot insertion.", "learning_hypothesis": "Reusable routines for thin-object pickup, regrasping, and slot insertion. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/deposit_coin.py", "sha256": "f9868ef827ec17b8db6c4650cc5d9a9e593d20d00ee822e2212ce229ec01fb61", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "e4cede8a217a89dcace79b57f7ac154415c0ff088aafabc35612ff514e009ec1", "wall_time_s": 60.23394230008125 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 129.84, "step_limit": 20000, "steps": 3246, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 134, "model_tool_calls": 133, "measured_protocol": { "episode_specs": [ { "instruction_override": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "deposit_coin" }, "backend_task": "robodojo.deposit_coin", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/deposit_coin", "instruction": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-09-candidate01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 9, "native_task_id": "robodojo/fasten_screws", "catalog_path": "tasks/task04/catalog/09-robodojo-fasten-screws/task.yaml", "catalog_sha256": "e640e9946e1614e178e6410328c8db2292b4596ad9096ed1b85a3bf83b771ad7", "current_instruction": "Fit all three nuts onto the upright bolts of the same color. Leave each nut upright, centered on its bolt, and fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "measured_instruction": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-10-candidate01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-10-candidate01", "previous_slot": 10, "recommendation": "Retain with stated limits", "native_conditions": "Three same-color nut/bolt matches; nuts upright and centered within 1 mm, seated deeper than 9 mm; grippers open and arms home.", "finding": "Retain with the existing observation caveat and explicit prior user selection. This is nut-on-bolt seating, not tightening torque or thread rotation. Red and blue assemblies pass; pink depth is only about 4.497 mm. Historical blue-object camera inconsistency is not repaired by instruction edits. Keep the current corrected instruction unchanged.", "structural_difficulty": "Color matching and precise multi-object assembly.", "learning_hypothesis": "Reusable routines for color matching and precise multi-object assembly. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/fasten_screws.py", "sha256": "2e187a28e33e44efb87baae6463b3ec35264ccd41e35c21a1be44f6361675185", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "48ec8b603e484e26832d63d493d66605a34af0a50a02e261424b1c36559de6cd", "wall_time_s": 61.955387028865516 }, "native_reward": 0.5, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 138.36, "step_limit": 20000, "steps": 3459, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 89, "model_tool_calls": 88, "measured_protocol": { "episode_specs": [ { "instruction_override": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "fasten_screws" }, "backend_task": "robodojo.fasten_screws", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/fasten_screws", "instruction": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-10-candidate01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 10, "native_task_id": "robodojo/play_stacking_toy", "catalog_path": "tasks/task04/catalog/10-robodojo-play-stacking-toy/task.yaml", "catalog_sha256": "cd1ca677defc449254f2489e332021d8749420ea1203ea17ecd3561f32c65a3b", "current_instruction": "Stack all ten toy pieces on the four pegs, grouping matching shapes together: four pieces on the tallest peg, three on the next tallest, two on the next, and one on the shortest. Center each piece precisely on its peg and lower it into the stack, then return both arms to their starting poses.", "measured_instruction": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-11-hard01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-11-hard01", "previous_slot": 11, "recommendation": "Retain provisionally", "native_conditions": "Ten pieces in groups of 4/3/2/1 on their designated pegs, anchor and successive-piece centering within 1 mm, sufficient insertion depth, arms home.", "finding": "The saved-layout shape counts and peg heights support the mapping. Preserve all four group sizes, descending height assignment and precise seating; delete redundant restatement that pieces stay on the toy. The native success already requires every group, not merely partial credit.", "structural_difficulty": "Long assembly sequence, shape grouping, recovery, and precise stacking.", "learning_hypothesis": "Reusable routines for long assembly sequence, shape grouping, recovery, and precise stacking. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/play_stacking_toy.py", "sha256": "23361329ee7efe330da1f1c241415205bafed2b6c082a913659d868a80d338c3", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "3fe1cdb26f6ebaa300edb45f1e2bb34d7e8aa2a734019a7c58cb2e1c52fa9ebc", "wall_time_s": 61.61338068265468 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 371.88, "step_limit": 20000, "steps": 9297, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 134, "model_tool_calls": 133, "measured_protocol": { "episode_specs": [ { "instruction_override": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "play_stacking_toy" }, "backend_task": "robodojo.play_stacking_toy", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/play_stacking_toy", "instruction": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-11-hard01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 1, "native_task_id": "behavior/assembling_gift_baskets", "catalog_path": "tasks/task05/catalog/01-behavior-assembling-gift-baskets/task.yaml", "catalog_sha256": "720d0bacf36fb2b9e7c1d826c57d3cd9015de46a22817ec78fb27c2f2dd18a49", "current_instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "measured_instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-01-completion02-kinex-v0.9.0-episode_01", "measured_task_id": "task05-01-completion02", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "One candle, cheese, cookie and bow in each of four baskets, via four BDDL pairing goals.", "finding": "Native instructions and initial arrangement match the 16-object allocation. The run leaves the required distribution unfinished. Keep the native instruction rather than adding grasping recipes.", "structural_difficulty": "Large object count and balanced allocation across four receptacles.", "learning_hypothesis": "Reusable routines for large object count and balanced allocation across four receptacles. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/assembling_gift_baskets/problem0.bddl", "sha256": "6e6c82bae96d3b4efaf3026ed84c2e0ea1aefa57e2f4763c51d2c2fe7b96267e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "assembling_gift_baskets", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_assembling_gift_baskets_instances/house_double_floor_lower_task_assembling_gift_baskets_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "98c28b1e404021338aabc713f9f2bae257c01bf54242e66afb9321b963893cc4", "split": "train" }, "loaded_state_sha256": "53e2563c4b2f5f77f0a5aa75b3ab6885eb71a2b590d087fbb19579a93e91d344", "native_clock_start_s": 0.32500001695007086, "native_context_start_index": 39, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 509.1397671867162 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 552.1666666666666, "step_limit": 50000, "steps": 16565, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 190, "model_tool_calls": 189, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "max_steps": 50000, "seed": 0, "task": { "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "assembling_gift_baskets", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.assembling_gift_baskets", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/assembling_gift_baskets", "instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-01-completion02/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 30.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 2, "native_task_id": "behavior/can_meat", "catalog_path": "tasks/task05/catalog/02-behavior-can-meat/task.yaml", "catalog_sha256": "4a67bffba57810f502c28a62ff8a89e376616ba96cf8641f77353d7f92c9686c", "current_instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "measured_instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-02-kinex-v0.8.2-episode_01", "measured_task_id": "task05-02", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Two cooked bratwursts in each of two closed jars; jars in cabinet; cabinet closed.", "finding": "The sausages are initially cooked; do not add a new cooking goal. The run does not complete the jar packing. Cabinet access and physical grasping are difficulty factors, not evidence of semantic mismatch.", "structural_difficulty": "Nested containment, exact counts, lids, and cabinet closure.", "learning_hypothesis": "Reusable routines for nested containment, exact counts, lids, and cabinet closure. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/can_meat/problem0.bddl", "sha256": "1862f9bb2fdeb8f528b499e4cf6167aa188321032cf57a1f072de13808383505", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "can_meat", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_can_meat_instances/house_single_floor_task_can_meat_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "b9f2b513b53f96ee0f4f3d75594907b2e343d47aaccf8d520bd4197bdea05534", "split": "train" }, "loaded_state_sha256": "b16803dc3264f390b148d5b51291454b27e452b5878dccc7be99df5adf050e1d", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 390.7425012663007 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 181.53333333333333, "step_limit": 50000, "steps": 5446, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 120, "model_tool_calls": 119, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "can_meat", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.can_meat", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/can_meat", "instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-02/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 3, "native_task_id": "behavior/moving_boxes_to_storage", "catalog_path": "tasks/task05/catalog/03-behavior-moving-boxes-to-storage/task.yaml", "catalog_sha256": "971459d6d2b9c79707c0a7e74e18c006ecceb0f101d3b326754bf13e4e7fc8c8", "current_instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "measured_instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-03-scene01-kinex-v0.8.2-episode_01", "measured_task_id": "task05-03-scene01", "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "Either box on the garage floor with the other on top of it.", "finding": "The native disjunction allows either stacking order, matching the original instruction. The selected run uses the corrected house_double_floor_lower scene and achieves native success. Do not reuse the superseded scene-mismatch trial.", "structural_difficulty": "Navigation with large carried objects and stacking.", "learning_hypothesis": "Reusable routines for navigation with large carried objects and stacking. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/moving_boxes_to_storage/problem0.bddl", "sha256": "13c91058b0266345eec5a5019b3882f1a1ff794dec5643548b2d42927b700995", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "moving_boxes_to_storage", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_moving_boxes_to_storage_instances/house_double_floor_lower_task_moving_boxes_to_storage_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "cda5caacc8efe2e62d8645de305d8cb6162ed79c8af3c862cfdbf58fd4aa9ac7", "split": "train" }, "loaded_state_sha256": "3bbc1544b2a23a21cb4c171ed06bc2d18339c9960dfd500eb5e407068fa59504", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 404.5288652488962 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 1109.3, "step_limit": 50000, "steps": 33279, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 339, "model_tool_calls": 341, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "moving_boxes_to_storage", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.moving_boxes_to_storage", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/moving_boxes_to_storage", "instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-03-scene01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 4, "native_task_id": "behavior/clean_up_your_desk", "catalog_path": "tasks/task05/catalog/04-behavior-clean-up-your-desk/task.yaml", "catalog_sha256": "f0921d8858e28b67c2f33f684835b1dc8475f271707de7074029cf5f42404b19", "current_instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "measured_instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-04-kinex-v0.8.2-episode_01", "measured_task_id": "task05-04", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Folders and books inside bookcase; two pens and pencil in pencil box; stapler, pencil box and closed laptop on desk.", "finding": "The object categories and fixtures match the original text. The existing run leaves organization unfinished; chair/access obstruction does not justify rewriting destinations.", "structural_difficulty": "Multi-category cleanup, small-object placement, and articulated laptop handling.", "learning_hypothesis": "Reusable routines for multi-category cleanup, small-object placement, and articulated laptop handling. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/clean_up_your_desk/problem0.bddl", "sha256": "c81e592bf14dd7b2fbbc2a443a5f4989e5d0d038f20c27cc91a4339b3d14318e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "clean_up_your_desk", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_clean_up_your_desk_instances/house_single_floor_task_clean_up_your_desk_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "a2ad7b92e2cc970ea6aebab530ee8b154cf603c92a487b0e16777e4110d8c202", "split": "train" }, "loaded_state_sha256": "49cbfd7e726d3c2d553c1a75f2c1dff481954da5b2f1c7e79d3d52cb00645201", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 391.4239665251225 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 467.8666666666667, "step_limit": 50000, "steps": 14036, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 288, "model_tool_calls": 304, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "clean_up_your_desk", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.clean_up_your_desk", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/clean_up_your_desk", "instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-04/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 5, "native_task_id": "behavior/setting_the_fire", "catalog_path": "tasks/task05/catalog/05-behavior-setting-the-fire/task.yaml", "catalog_sha256": "dcab398c37562863d13141b5db4ef14140d63bc97565270506d5d84f67899867", "current_instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "measured_instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-05-kinex-v0.8.2-episode_01", "measured_task_id": "task05-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Both firewood pieces burning, on the newspaper and inside fireplace; newspaper burning; lighter off.", "finding": "The original wording captures placement, ignition and extinguishing the lighter. Burning wood outside the fireplace does not satisfy the goal. BDDL checks wood-in-fireplace explicitly; newspaper location is partly mediated by the wood-on-paper relation.", "structural_difficulty": "Relational arrangement and coupled material state changes.", "learning_hypothesis": "Reusable routines for relational arrangement and coupled material state changes. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/setting_the_fire/problem0.bddl", "sha256": "d7d62278c4be89fa1199239267784bbcbc87270a5fb64a6a9705ecfff4525c96", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "setting_the_fire", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_setting_the_fire_instances/house_double_floor_lower_task_setting_the_fire_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "534427186b9394df338a54b4a7a6004edbf0b7c0aff5ca6213da9123e27f6fcb", "split": "train" }, "loaded_state_sha256": "898740f9b4bc509c34665aa782a1f8fb0c645998eb553e9c6d4799b519517ed6", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 274.6435562046245 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 703.2, "step_limit": 50000, "steps": 21096, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 381, "model_tool_calls": 380, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "setting_the_fire", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.setting_the_fire", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/setting_the_fire", "instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 6, "native_task_id": "behavior/clean_a_trumpet", "catalog_path": "tasks/task05/catalog/06-behavior-clean-a-trumpet/task.yaml", "catalog_sha256": "5d397a071752e95a2cc6cf534184e8e1025642d3f8ef05a144ee05ab18118e19", "current_instruction": "In the bedroom, use the scrub brush from the desk to remove all dust from the cornet (trumpet) on the desk.", "measured_instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task05-06-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task05-06-completion01", "previous_slot": 6, "recommendation": "Retain with stated limits", "native_conditions": "The cornet is not covered with dust; the visual-particle threshold makes this zero attached dust particles.", "finding": "Retain provisionally with visibility uncertainty. Latest complete evidence leaves 18 of 20 particles, so this is not a near-complete threshold-only failure. The older 19-of-20-cleaned interrupted run is not substituted. Keep 'all dust'; remove suggested turning and multi-view procedure. Visibility/reachability of the last particle across instances remains unproved.", "structural_difficulty": "Contact cleaning of a complex curved object and inspection.", "learning_hypothesis": "Reusable routines for contact cleaning of a complex curved object and inspection. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/clean_a_trumpet/problem0.bddl", "sha256": "3d6019ad5d1f0e4e110217bdba0b43bd4ca0757ad9daae6e873ae5cfa8ffdedb", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "clean_a_trumpet", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_upper_task_clean_a_trumpet_instances/house_double_floor_upper_task_clean_a_trumpet_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "5395791af3c42e17a2917bf6f213017603434359c07da3df0b18845aff24bc85", "split": "train" }, "loaded_state_sha256": "acc6ed16c6e3706038fce9f5731ba75fa9b99d0c5f330c34ec3ee36e63e5ca1c", "native_clock_start_s": 0.32500001695007086, "native_context_start_index": 39, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_upper", "seed_selects_instance": false, "wall_time_s": 372.69310517050326 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 869.9, "step_limit": 50000, "steps": 26097, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 323, "model_tool_calls": 322, "measured_protocol": { "episode_specs": [ { "instruction_override": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "max_steps": 50000, "seed": 0, "task": { "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "clean_a_trumpet", "max_steps": 50000, "scene_model": "house_double_floor_upper", "stop_on_success": true }, "backend_task": "behavior.clean_a_trumpet", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/clean_a_trumpet", "instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-06-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 30.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 7, "native_task_id": "behavior/slicing_vegetables", "catalog_path": "tasks/task05/catalog/07-behavior-slicing-vegetables/task.yaml", "catalog_sha256": "30a114db47d6a36eb4c1f97ae560cb0fc2c1dd3dafd0153bf118c1b8be5cdb76", "current_instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "measured_instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-07-kinex-v0.8.2-episode_01", "measured_task_id": "task05-07", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Diced systems for all three vegetable types are real, all five original vegetables are no longer real, and refrigerator closed.", "finding": "The original instruction explicitly asks to dice all five vegetables and close the refrigerator. The run fails during access before completing the transformations. Sampled camera frames do not expose the still-enclosed food; scene identities also come from BDDL and loaded-instance evidence.", "structural_difficulty": "Fixture access, tool contact, and object transformation.", "learning_hypothesis": "Reusable routines for fixture access, tool contact, and object transformation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/slicing_vegetables/problem0.bddl", "sha256": "783060c491bc4ec9cd9ad2bb2e58331ccc895fb4b0f0af418f9e2437135b15d0", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "slicing_vegetables", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_slicing_vegetables_instances/house_single_floor_task_slicing_vegetables_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "9a46a9096e953ae8eadba3bccf58c5658376ea759ea2284723ca55f7821372cd", "split": "train" }, "loaded_state_sha256": "409b3591247242b9b26db777a53c7bcfa6e94f5890e4ec91588699967e8f0870", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 399.51723534055054 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 106.06666666666666, "step_limit": 50000, "steps": 3182, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 79, "model_tool_calls": 78, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "slicing_vegetables", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.slicing_vegetables", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/slicing_vegetables", "instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-07/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 8, "native_task_id": "behavior/freeze_pies", "catalog_path": "tasks/task05/catalog/08-behavior-freeze-pies/task.yaml", "catalog_sha256": "12b0e87ddd219b430aa1a58a2e379b1ee4fb04543e2da5b65e3d1488d896f4d0", "current_instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "measured_instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-08-recovery01-kinex-v0.8.2-episode_01", "measured_task_id": "task05-08-recovery01", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "Each pie in a separate container; both containers in refrigerator; both pies frozen; refrigerator closed.", "finding": "Keep the original packing and freezing goal. Frozen depends on temperature, not mere fridge containment. The terminal pies remain about 23 C and the manipulation goal is incomplete; this does not diagnose a cooling defect. Startup was completed in the selected run.", "structural_difficulty": "Nested containment, navigation, fixture closure, and delayed temperature change.", "learning_hypothesis": "Reusable routines for nested containment, navigation, fixture closure, and delayed temperature change. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/freeze_pies/problem0.bddl", "sha256": "78671ab6e34b48033a01ea4a97ba072d1268935fff11eb71bf8a200685261870", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "freeze_pies", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_freeze_pies_instances/house_single_floor_task_freeze_pies_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "7b1aa7d5c7958488798e17b544ae3d4fa97d3e2250f69d66ea93cc2c3e327931", "split": "train" }, "loaded_state_sha256": "031ba3b5f8f6858abb3bc662bf7ffb6fbb9889c9c93ed4a20f1c6948867ad15e", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 378.04605627898127 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 1238.7, "step_limit": 50000, "steps": 37161, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 451, "model_tool_calls": 465, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "freeze_pies", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.freeze_pies", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/freeze_pies", "instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-08-recovery01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false } ], "contact_candidate_review": "contact-candidates.json", "hard_candidate_review": "hard-candidates.json", "retained_instruction_revisions": "retained-instruction-revisions.json", "retained_by_user_with_caveats": [ "robodojo/fasten_screws: historical camera-observation caveat remains unresolved." ], "scanner_review": "scanner-review.json" }