{ "schema": "rlebench/instruction-audit/1", "historical_audited_at": "2026-10-01T18:51:02.439991+00:00", "historical_native_failures": 18, "selected_instruction_reruns": 13, "historical_revised_catalog_not_used": [ "task01/03", "task01/04", "task01/05", "task01/07", "task03/02", "task04/07" ], "historical_modified_runs_with_old_describe": [ "task01/06", "task01/08", "task04/04" ], "historical_tasks_requiring_extra_alignment_review": [ "task01/08", "task04/04" ], "scope": "Historical observations are preserved; current observations come from the new verified episodes.", "episodes": [ { "slot": "task01/01", "native_task_id": "robocasa/LoadCondimentsInFridge", "historical_episode_id": "task01-01-recovery01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves.", "catalog_instruction_at_historical_audit": "Load the condiments into the fridge following the current scene instruction from roboenv describe.", "historical_describe_instructions": [ "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "The replay ends with condiment bottles on the floor rather than both on the top rack. The agent reports failed grasps and a dropped mustard bottle.", "historical_instruction_verifier_alignment": "The current scene-based instruction describes storage but omits the final gripper-distance requirement. This omission did not cause this episode to fail because the storage goal itself was not met.", "whole_episode_false_negative_proven": false, "native_source_sha256": "374a8bf1a2a0b7ec8eb6df4c471b162d9c8e156d35d28580afdb6cc574807b3d", "selected_for_instruction_rerun": true, "current_episode_id": "task01-01-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 155.9, "step_limit": 20000, "steps": 3118, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "The condiments were not stored in the fridge.", "observation": "The verified terminal frame shows both condiments on the floor outside the fridge. The recorded session also reports that neither could be recovered. The top-shelf storage goal remains unmet.", "evidence": [ "Native terminal verdict", "Verified recorded integration-state playback", "Recorded agent session" ] } }, { "slot": "task01/02", "native_task_id": "robocasa/FilterMicrowavableItem", "historical_episode_id": "task01-02-retry01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak.", "catalog_instruction_at_historical_audit": "Prepare the bowl for microwaving following the current scene instruction from roboenv describe.", "historical_describe_instructions": [ "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "The mango reaches the plate, but the bowl falls and the steak is not microwaved inside it. Terminal video does not show the required closed, operating microwave setup.", "historical_instruction_verifier_alignment": "Main instruction and object-selection goal agree. Final gripper clearance is omitted from the native wording and current scene wrapper; this is a latent mismatch, not the observed failure cause.", "whole_episode_false_negative_proven": false, "native_source_sha256": "89ef510ac1ad0e1fd13094e477a1bed00a2361a351625778fe8cd8289aeab594", "selected_for_instruction_rerun": true, "current_episode_id": "task01-02-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 297.4, "step_limit": 20000, "steps": 5948, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "The steak was not microwaved.", "observation": "The mango is on the plate, but the bowl is on the floor and the steak remains outside the microwave. The recorded session reports that the fallen bowl could not be recovered.", "evidence": [ "Native terminal verdict", "Verified recorded integration-state playback", "Recorded agent session" ] } }, { "slot": "task01/03", "native_task_id": "robocasa/StoreDumplings", "historical_episode_id": "task01-03-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Place two dumplings into each of the tupperware containers and then place the containers in the fridge.", "catalog_instruction_at_historical_audit": "Place two dumplings side by side in each food container. Put both filled containers in the fridge, then release them and withdraw the gripper.", "historical_describe_instructions": [ "Place two dumplings into each of the tupperware containers and then place the containers in the fridge." ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "One container is moved into the fridge, while another container and food remain on the floor. The full two-container, four-dumpling storage goal is not completed.", "historical_instruction_verifier_alignment": "The authored revised instruction adds side-by-side placement, release and withdrawal, but the measured run used the shorter native instruction. Current withdrawal wording still does not specify clearance from every relevant object.", "whole_episode_false_negative_proven": false, "native_source_sha256": "666a531feb0a6a250953ccb0eebf32b72d763660aecce198b504dc722593ae77", "selected_for_instruction_rerun": true, "current_episode_id": "task01-03-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "current_instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 830.75, "step_limit": 20000, "steps": 16615, "success": false, "usage_complete": true, "valid": true }, "current_review": { "summary": "Both filled food containers were not stored in the fridge.", "observation": "In the explicitly authorized replacement, all four dumplings and both containers end near floor height (roughly 2\u20133 cm for their body origins), outside the fridge shelves. Recorded integration-state playback and independent terminal geometry agree. The storage and food-count goal is incomplete, rather than rejected solely for withdrawal.", "evidence": [ "Native terminal verdict", "Every recorded integration state restored with zero difference", "Terminal body positions from the frozen MuJoCo model" ], "recommendation": "Retain provisionally" } }, { "slot": "task01/04", "native_task_id": "robocasa/DivideBuffetTrays", "historical_episode_id": "task01-04-retry01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray.", "catalog_instruction_at_historical_audit": "Place the two food groups named in the scene instruction on separate dining-counter trays, one group per tray. Release the food and withdraw the gripper.", "historical_describe_instructions": [ "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray." ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "Food remains in the fridge or falls outside the trays. Neither the scene nor the transcript establishes that both groups were completely placed on their respective trays.", "historical_instruction_verifier_alignment": "The measured run lost the catalog release/withdrawal clarification. Current wording mentions food clearance, while the native predicate also checks both trays.", "whole_episode_false_negative_proven": false, "native_source_sha256": "ecd23b0480a93254ea14f70d3141d450479f83df5c093f41a09cdcad90f1b545", "selected_for_instruction_rerun": true, "current_episode_id": "task01-04-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "current_instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 783.65, "step_limit": 20000, "steps": 15673, "success": false, "usage_complete": true, "valid": true }, "current_review": { "summary": "The food groups remain in the fridge instead of reaching the two trays.", "observation": "The authorized replacement ends with vegetables and meat at the fridge shelves. Their terminal XY positions are several metres from the two dining-counter trays. Both food-group placement requirements remain incomplete. Usage is complete (210 prepared requests, 210 raw completed responses); the interrupted original is retained separately.", "evidence": [ "Native terminal verdict", "Verified recorded integration-state playback", "Independent terminal object positions" ], "recommendation": "Retain provisionally" } }, { "slot": "task01/05", "native_task_id": "robocasa/MakeCheesecakeFilling", "historical_episode_id": "task01-05-recovery01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Add the butter stick, sugar cube, and cream cheese stick to the stand mixer bowl and then turn the speed knob to begin making cheesecake filling.", "catalog_instruction_at_historical_audit": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "historical_describe_instructions": [ "Add the butter stick, sugar cube, and cream cheese stick to the stand mixer bowl and then turn the speed knob to begin making cheesecake filling." ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "Ingredients are not all in the mixer bowl: the video shows food remaining outside or falling during manipulation. This is not a completed filling task rejected solely for the mixer head.", "historical_instruction_verifier_alignment": "The old measured instruction omitted lowering the mixer head. The current catalog explicitly supplies that missing condition, but this run did not receive it.", "whole_episode_false_negative_proven": false, "native_source_sha256": "905f1185f5f72e42bb14ed4e8f8afa282859d2b093e704743c2ff83cdf66b7ad", "selected_for_instruction_rerun": true, "current_episode_id": "task01-05-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 730.05, "step_limit": 20000, "steps": 14601, "success": false, "usage_complete": true, "valid": true }, "current_review": { "summary": "The mixer was not turned on.", "observation": "The recorded mixer speed joint is approximately -0.0000007, whereas the native check requires its speed value above 0.2. The revised instruction explicitly requires lowering the head and turning the mixer on. This attempt does not demonstrate a completed mixing goal falsely rejected for a hidden head condition.", "evidence": [ "Native terminal verdict", "Recorded terminal mixer joints", "Pinned native success definition" ], "recommendation": "Retain provisionally" } }, { "slot": "task01/06", "native_task_id": "robocasa/MultistepSteaming", "historical_episode_id": "task01-06-instruction01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Rinse the lemon wedge under running water in the sink. Turn off the faucet, transfer that same lemon wedge to the pot next to the stove, and place the pot on the rear left burner.", "catalog_instruction_at_historical_audit": "Rinse the food item named in the current scene instruction under running water in the sink. Turn off the faucet, transfer that same item to the pot, and place the pot on the specified burner.", "historical_describe_instructions": [ "Turn on the sink faucet. Then move the lemon wedge from the counter to the sink. Turn off the sink. Move the vegetable from the sink to the pot next to the stove. Finally move the pot to the rear left burner." ], "historical_provenance_assessment": "revised_instruction_run", "historical_terminal_observation": "The lemon wedge falls to the floor during the attempt. Moving the pot to the stove does not establish that the food was rinsed and transferred into it.", "historical_instruction_verifier_alignment": "The revised effective wording agrees with the native sequence. The agent also sees the older instruction through describe. Physical placement inside the sink, rather than merely near the water stream, is needed.", "whole_episode_false_negative_proven": false, "native_source_sha256": "e745045ff2b19773ca6ff33fd3485daad38266bb772d06ea5f112c3ad8eff94d", "selected_for_instruction_rerun": true, "current_episode_id": "task01-06-completion01-kinex-v0.9.0-episode_01", "current_instruction": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 464.4, "step_limit": 20000, "steps": 9288, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "Completed revised-instruction evidence; the native task remains unsuccessful.", "observation": "Native validity, normal execution, full provider usage and replay are checked separately. There is no evidence here of a fully satisfied physical goal rejected solely by the verifier.", "recommendation": "Retain provisionally", "evidence": [ "Native verifier", "Complete provider audit", "Matched actual instruction surfaces", "Verified replay" ] } }, { "slot": "task01/07", "native_task_id": "robocasa/ScalePortioning", "historical_episode_id": "task01-07-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Take the chicken drumstick from the fridge and place it on the digital scale on the counter by the fridge. Wait a few seconds for a reading, then move it to the plate on the dining counter.", "catalog_instruction_at_historical_audit": "Place the specified food from the fridge on the digital scale and release it. Wait a few seconds, then move it onto the dining-counter plate and withdraw the gripper.", "historical_describe_instructions": [ "Take the chicken drumstick from the fridge and place it on the digital scale on the counter by the fridge. Wait a few seconds for a reading, then move it to the plate on the dining counter." ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "The chicken drumstick falls before successful weighing and final plating. The measured attempt never demonstrates the full weighing-and-transfer sequence.", "historical_instruction_verifier_alignment": "The measured old instruction does not require releasing and withdrawing while waiting. The revised catalog fixes release, but its scale-stage wording can still explicitly say to move the gripper away during the wait.", "whole_episode_false_negative_proven": false, "native_source_sha256": "50b6fa4bd2415d18ad6cb3e4f9ac7021cfdc47d8720b5700c1a4e97156708579", "selected_for_instruction_rerun": true, "current_episode_id": "task01-07-completion01-kinex-v0.9.0-episode_01", "current_instruction": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 243.9, "step_limit": 20000, "steps": 4878, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "Completed revised-instruction evidence; the native task remains unsuccessful.", "observation": "Native validity, normal execution, full provider usage and replay are checked separately. There is no evidence here of a fully satisfied physical goal rejected solely by the verifier.", "recommendation": "Retain provisionally", "evidence": [ "Native verifier", "Complete provider audit", "Matched actual instruction surfaces", "Verified replay" ] } }, { "slot": "task01/08", "native_task_id": "robocasa/ScrubCuttingBoard", "historical_episode_id": "task01-08-instruction01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Clean the cutting board by scrubbing across its surface with the sponge. When finished, release the sponge and move the gripper clear of it.", "catalog_instruction_at_historical_audit": "Clean the cutting board by scrubbing across its surface with the sponge. When finished, release the sponge and move the gripper clear of it.", "historical_describe_instructions": [ "Pick up the sponge from the counter and clean the cutting board by briefly scrubbing or pressing down on the cutting board. Once finished, release the sponge." ], "historical_provenance_assessment": "revised_instruction_run", "historical_terminal_observation": "The recorded-state audit reconstructs four accepted locations and an 8.30-8.40 cm span. The sponge is released, but final gripper distance is 13.759 cm. Of 70 board-contact frames with bilateral finger contact, 36 are rejected by the native grasp heuristic; even all such contact locations span only 9.322 cm.", "historical_instruction_verifier_alignment": "The revised wording removes stationary pressing, but generic scrubbing and moving clear still leave the coverage/count/clearance requirements unclear. Describe exposes the older wording allowing brief pressing. The grasp heuristic is a potential false-negative source; bilateral contact alone does not prove a stable grasp. Relaxing that heuristic alone does not make this measured episode pass.", "whole_episode_false_negative_proven": false, "native_source_sha256": "3a4082816ead472ea533bad42fe11a198c8baf10ded05abeef0fbb58e1b1e3e0", "selected_for_instruction_rerun": true, "current_episode_id": "task01-08-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "current_native_verdict": { "episode": 1, "initialization": null, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 75.2, "step_limit": 20000, "steps": 1504, "success": true, "usage_complete": true, "valid": true }, "current_review": { "summary": "The board-cleaning task now satisfies the unchanged native check.", "observation": "The revised broad-sweep, surface-contact and withdrawal instruction produced native success after 1,504 action frames. All 46 prepared provider requests have raw terminal responses. The previous failure is not reproduced in this run; one successful seed does not prove the grasp heuristic correct in every case.", "evidence": [ "Native terminal success", "Complete independent provider audit", "Unchanged native predicate" ], "recommendation": "Retain provisionally" } }, { "slot": "task03/02", "native_task_id": "robotwin/robotwin-aloha.place_dual_shoes", "historical_episode_id": "task03-02-kinex-v0.8.2-episode_01", "historical_effective_instruction": "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left", "catalog_instruction_at_historical_audit": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "historical_describe_instructions": [ "use both arms to pick up the two shoes on the table and put them in the shoebox, with the shoe tip pointing to the left" ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "Both shoes reach the box region, but both fail their assigned XY checks. One shoe is 7.908 cm above its target height and fails orientation. The other passes height and orientation but is in the opposite assigned region.", "historical_instruction_verifier_alignment": "The old instruction does not assign shoe identities to front/back halves or explicitly require flat placement. The current catalog supplies these missing conditions, but it was overwritten before the measured run. The height/orientation error remains a real incomplete-goal condition even if identity assignment is ignored.", "whole_episode_false_negative_proven": false, "native_source_sha256": "f35c4a133e1de567b274cbd15f158f585eb05e32aeb05be88839d249d0d7ab9e", "selected_for_instruction_rerun": true, "current_episode_id": "task03-02-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "current_native_verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 5883, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "a715656d4d3fff3a16238f57b8b2cbe6574b5fc164600c21f74bf7c3f8156513", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 11485, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 43.20029250346124 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 74.12000352051109, "step_limit": 20000, "steps": 1853, "success": true, "valid": true, "usage_complete": true }, "current_review": { "summary": "Both shoes satisfy the native placement checks.", "observation": "Both shoes satisfy the native box position, height and orientation checks. Fresh-process replay reproduces the initial/final states, control targets, budget and successful verdict. The run stopped automatically when the placement predicate became true; the right gripper remained closed at that point.", "evidence": [ "Native terminal verdict", "Independent terminal geometry audit", "Verified native command replay" ] } }, { "slot": "task03/06", "native_task_id": "robotwin/robotwin-aloha.scan_object", "historical_episode_id": "task03-06-instruction01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Hold the tea box and scanner with separate grippers. Bring the scanner close and point its reading face directly at the center of the box's bottom face.", "catalog_instruction_at_historical_audit": "Hold the tea box and scanner with separate grippers. Bring the scanner close and point its reading face directly at the center of the box's bottom face.", "historical_describe_instructions": [ "Hold the tea box and scanner with separate grippers. Bring the scanner close and point its reading face directly at the center of the box's bottom face." ], "historical_provenance_assessment": "revised_instruction_run", "historical_terminal_observation": "Terminal geometry has a negative signed projection for all five possible scanner assets, ranging from -7.46 to -6.01 cm. All satisfy the perpendicular residual check, but all fail the directional/distance check. Exact scanner model ID is not preserved in the pose capture, so every candidate was evaluated.", "historical_instruction_verifier_alignment": "All tea-box assets place their bottom-face functional point at the body origin, so the revised bottom-face target does not disagree with the checker reference point. Facing the correct side of the reading head remains essential. This audit does not establish a completed scan falsely rejected.", "whole_episode_false_negative_proven": false, "native_source_sha256": "860dfe2cc86d180096089bc87502c7904f7ce2efb64fb32620c1720ca0178c2b", "selected_for_instruction_rerun": true, "current_episode_id": "task03-06-instruction02-kinex-v0.9.0-episode_01", "current_instruction": "Hold the tea box and scanner with separate grippers. Aim the scanner's optical reading window directly at the center of the box's bottom face, rather than its back or handle. Bring that window close to the bottom face with a small air gap, keeping both objects held and the reading window facing the box.", "current_native_verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 4522, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "6ce2d4764c6abe9d08b1eabef8ced34805aa4dc7bb02848bdbf86fca92112ed6", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 10124, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 68.65967309847474 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 153.16000727470964, "step_limit": 20000, "steps": 3829, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "The scanner alignment did not satisfy the native directional check.", "observation": "The box and scanner remain held close together in terminal playback. Terminal geometry passes perpendicular alignment, but the box lies on the opposite side of the annotated reading point from the required direction for all five possible scanner assets. Offline replay of 4,185 commands reproduces initial/final states, control targets, the 3,829-frame budget and the unsuccessful native verdict with zero state difference. This does not prove a verifier false negative.", "evidence": [ "Native terminal verdict", "Independent terminal geometry audit for all five scanner assets", "Exact image asset metadata", "Verified native command replay and three-camera terminal frames" ], "recording_limitation": "Original capture did not finalize. The derived video is verified native command replay; original per-frame states are unavailable. Raw recording evidence remains unchanged." } }, { "slot": "task04/04", "native_task_id": "robodojo/store_laptop_and_headphones", "historical_episode_id": "task04-04-instruction01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Hang the headphones by their headband on the stand. Close the laptop fully and place it upright in its vertical stand. Return both arms to their starting poses.", "catalog_instruction_at_historical_audit": "Hang the headphones by their headband on the stand. Close the laptop fully and place it upright in its vertical stand. Return both arms to their starting poses.", "historical_describe_instructions": [ "Hang the headphones on the headphone stand, close the laptop, then place it into the vertical laptop stand." ], "historical_provenance_assessment": "revised_instruction_run", "historical_terminal_observation": "Headphones are visibly hanging, and the headband bridge distance is 4.581 cm, which passes. Full orientation error is 53.943 degrees, which fails; native position-only helper returns true, while the orientation-enforced helper returns false. Independently, the laptop remains on the table, with hinge -0.769 rad, outside its storage rack.", "historical_instruction_verifier_alignment": "Hanging by the headband does not specify this additional cradle-relative orientation. This is an evidenced mismatch for a completed-looking subgoal, while the overall episode is still unsuccessful because laptop storage is plainly incomplete.", "whole_episode_false_negative_proven": false, "native_source_sha256": "2bee4e31f0594900629fe7f43266e16fecb310b152930ce51524507136809a92", "selected_for_instruction_rerun": true, "current_episode_id": "task04-04-completion01-kinex-v0.9.0-episode_01", "current_instruction": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "current_native_verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "d804229232414a2f4446497c48f57cc03621f2552f2c33ddf0705c5b2b3d39c5", "wall_time_s": 52.43044571578503 }, "native_reward": 0.2, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 781.36, "step_limit": 20000, "steps": 19534, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "Headset placement now passes the native position and orientation predicates.", "observation": "Both the position-only and orientation-enforced headset helpers return true. The laptop fails the upright-axis and bottom-in-rack predicates. The clarified headband/cradle instruction resolves the known headset subgoal mismatch in this seed; the whole task remains incomplete.", "recommendation": "Retain provisionally", "evidence": [ "Latest native terminal geometry", "Headset position and orientation predicate values", "Laptop upright and rack predicate values" ] } }, { "slot": "task04/07", "native_task_id": "robodojo/store_tools_in_toolbox", "historical_episode_id": "task04-07-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Place each tool into its matching position in the toolbox, then reset the robot arm.", "catalog_instruction_at_historical_audit": "Fit each tool fully into its matching recess in the toolbox, then return both arms to their starting poses.", "historical_describe_instructions": [ "Place each tool into its matching position in the toolbox, then reset the robot arm." ], "historical_provenance_assessment": "revised_catalog_overwritten_with_original_native", "historical_terminal_observation": "All four native slot predicates fail and all four containment predicates fail. Slot distances are 12.67 cm (hammer), 8.36 cm (pliers), 14.13 cm (tape measure), and 1.35 cm (wrench); even the close wrench fails orientation by 133.19 degrees. Tool upper bounds protrude 1.48-2.66 cm above the toolbox upper bound.", "historical_instruction_verifier_alignment": "Tools being inside the outer box does not mean they fit their matching recesses. The revised catalog makes full recess fitting clearer, but the measured run received the old instruction. The current evidence is not a near-boundary false failure caused solely by an overly strict tolerance.", "whole_episode_false_negative_proven": false, "native_source_sha256": "2969fc3eb171560b3c0bcea62b9f38b8b35f3aecc9bcc7085c3b1cec365b48fc", "selected_for_instruction_rerun": true, "current_episode_id": "task04-07-completion01-kinex-v0.9.0-episode_01", "current_instruction": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "current_native_verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "15029891c722a98c9f1b3c92d88dd5883af61f88169c15c35ff4194ea13e4527", "wall_time_s": 59.26869597937912 }, "native_reward": 0.5, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 773.6, "step_limit": 20000, "steps": 19340, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "The clarified recess-fitting requirement aligns with the native slot predicates.", "observation": "All four native slot predicates remain false. Tape measure and wrench pass full box containment, while hammer and pliers fail it. Loose box containment is insufficient for the explicitly requested matching recesses.", "recommendation": "Retain provisionally", "evidence": [ "Latest terminal poses", "Pinned slot and containment helpers" ] } }, { "slot": "task05/02", "native_task_id": "behavior/can_meat", "historical_episode_id": "task05-02-kinex-v0.8.2-episode_01", "historical_effective_instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "catalog_instruction_at_historical_audit": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "historical_describe_instructions": [ "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "Owner goal report marks only jar-inside-cabinet and jar-closed groups satisfied. Exact meat count and cabinet closure fail. The transcript and video do not establish meat being successfully transferred into the jars.", "historical_instruction_verifier_alignment": "The current measured instruction explicitly states the same counts, separate jars and closure requirements. No hidden stricter goal is identified.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": false }, { "slot": "task05/04", "native_task_id": "behavior/clean_up_your_desk", "historical_episode_id": "task05-04-kinex-v0.8.2-episode_01", "historical_effective_instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "catalog_instruction_at_historical_audit": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "historical_describe_instructions": [ "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "The only satisfied owner condition is pencil box on desk. Seven other goal groups remain unsatisfied, including laptop relocation/closure and stationery storage. The video shows the laptop ending away from the required desk position.", "historical_instruction_verifier_alignment": "Measured instruction and saved owner goal conditions agree. This is broad incomplete execution, not a fine geometric rejection of a completed desk cleanup.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": false }, { "slot": "task05/05", "native_task_id": "behavior/setting_the_fire", "historical_episode_id": "task05-05-kinex-v0.8.2-episode_01", "historical_effective_instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "catalog_instruction_at_historical_audit": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "historical_describe_instructions": [ "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "Newspaper burning and lighter-off predicates pass, but the firewood conjunction fails. Wood remains outside the completed fireplace pile and has not fully caught fire.", "historical_instruction_verifier_alignment": "The instruction explicitly requires the whole pile to catch fire and both pieces to be in the fireplace. Burning only the newspaper is insufficient under the stated task.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": false }, { "slot": "task05/06", "native_task_id": "behavior/clean_a_trumpet", "historical_episode_id": "task05-06-kinex-v0.8.2-episode_01", "historical_effective_instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk until it's no longer covered in dust.", "catalog_instruction_at_historical_audit": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk until it's no longer covered in dust.", "historical_describe_instructions": [ "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk until it's no longer covered in dust." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "Recorded owner state contains 20 initial dust particles and 8 final particles attached to trumpet_87. The brush ModifiedParticles counter records 12 removed particles. The native goal remains false. The trumpet and brush end on the floor.", "historical_instruction_verifier_alignment": "The agent made measurable cleaning progress but did not finish. The phrase no longer covered in dust is weaker than the checker zero-particle interpretation; remaining dust is substantial here (40% of initial particles), so the episode is not rejected solely for one tiny leftover spot.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": true, "current_episode_id": "task05-06-completion01-kinex-v0.9.0-episode_01", "current_instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "current_native_verdict": { "episode": 1, "initialization": { "activity": "clean_a_trumpet", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_upper_task_clean_a_trumpet_instances/house_double_floor_upper_task_clean_a_trumpet_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "5395791af3c42e17a2917bf6f213017603434359c07da3df0b18845aff24bc85", "split": "train" }, "loaded_state_sha256": "acc6ed16c6e3706038fce9f5731ba75fa9b99d0c5f330c34ec3ee36e63e5ca1c", "native_clock_start_s": 0.32500001695007086, "native_context_start_index": 39, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_upper", "seed_selects_instance": false, "wall_time_s": 372.69310517050326 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 869.9, "step_limit": 50000, "steps": 26097, "success": false, "valid": true, "usage_complete": true }, "current_review": { "summary": "The latest completed run leaves eighteen attached dust particles.", "observation": "Native particle groups contain20 initial and18 final dust particles attached to trumpet_87. The instrument ends near floor height. The revised instruction explicitly requests all dust specks to be removed; this failure is far from completion and does not establish a hidden last-particle problem. The earlier19/20-cleaned attempt remains an interrupted, excluded record.", "recommendation": "Retain provisionally; inspect multi-view observability before any stronger last-particle claim", "evidence": [ "Initial/final native dust groups", "Complete provider audit", "Historical interrupted attempt" ] } }, { "slot": "task05/07", "native_task_id": "behavior/slicing_vegetables", "historical_episode_id": "task05-07-kinex-v0.8.2-episode_01", "historical_effective_instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "catalog_instruction_at_historical_audit": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "historical_describe_instructions": [ "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "Only the fridge-closed condition passes. No required diced instances exist, and the original vegetables remain. The agent does not complete retrieval or cutting.", "historical_instruction_verifier_alignment": "The measured instruction explicitly specifies dicing every vegetable and leaving only diced pieces. No stricter hidden condition is identified.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": false }, { "slot": "task05/08", "native_task_id": "behavior/freeze_pies", "historical_episode_id": "task05-08-recovery01-kinex-v0.8.2-episode_01", "historical_effective_instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "catalog_instruction_at_historical_audit": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "historical_describe_instructions": [ "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen." ], "historical_provenance_assessment": "current_instruction_semantically_unchanged", "historical_terminal_observation": "All four owner goal groups are unsatisfied. Pies remain essentially at their initial counter/plate positions, outside the required packed refrigerator state, and are not frozen.", "historical_instruction_verifier_alignment": "The measured instruction explicitly includes packing, refrigerator placement, closure and waiting until frozen. This run never reaches the packing stage; it cannot demonstrate that the frozen threshold is too strict or unreachable.", "whole_episode_false_negative_proven": false, "native_source_sha256": null, "selected_for_instruction_rerun": false } ], "current_audited_at": "2026-10-05T18:02:45.547675+00:00", "current_tasks_requiring_extra_alignment_review": [], "current_describe_mismatch_observed": false, "current_scope": "Current retained episodes and their instruction alignment checks. Removed tasks are excluded.", "decisions": { "retain_provisionally": [ "task01/08", "task04/04", "task04/07", "task05/06" ], "unresolved_calibration": [], "running_evidence_pending": [], "proven_whole_task_oracle_only_false_negatives": [] }, "contact_candidates": [ { "native_task_id": "robodojo/push_T", "native_instruction": "Push the T-shaped block to align it precisely with the gray T-shaped pad.", "effective_instruction": "Push the T-shaped block along the tabletop without lifting it, until its position and orientation align precisely with the gray T-shaped pad. Then return both arms to their starting poses.", "instruction_policy": "modified_native", "episode": "task04-08-candidate01-kinex-v0.9.0-episode_01" }, { "native_task_id": "robodojo/deposit_coin", "native_instruction": "Pick up the coin from the holder and insert it precisely into the coin bank.", "effective_instruction": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "instruction_policy": "modified_native", "episode": "task04-09-candidate01-kinex-v0.9.0-episode_01" }, { "native_task_id": "robodojo/fasten_screws", "native_instruction": "Insert and tighten each screw into the nut of the same color.", "effective_instruction": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "instruction_policy": "modified_native", "episode": "task04-10-candidate01-kinex-v0.9.0-episode_01" } ], "hard_candidates": [ { "native_task_id": "robodojo/insert_tubes", "native_instruction": "Insert the three tubes into the rack one by one.", "effective_instruction": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "instruction_policy": "modified_native", "episode": "task04-08-hard02-kinex-v0.9.0-episode_01" }, { "native_task_id": "robodojo/play_stacking_toy", "native_instruction": "Place all stacking toy pieces onto the correct pegs.", "effective_instruction": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "instruction_policy": "modified_native", "episode": "task04-11-hard01-kinex-v0.9.0-episode_01" } ], "scanner_rerun": { "native_task_id": "robotwin/robotwin-aloha.scan_object", "native_instruction": "simultaneously pick up the scanner and the object with separate arms, then scan the object", "effective_instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "episode": "task03-06-instruction03-kinex-v0.9.0-episode_01", "instruction_policy": "modified_native" }, "current_selection": [ { "family": "task01", "slot": 1, "native_task_id": "robocasa/LoadCondimentsInFridge", "catalog_path": "tasks/task01/catalog/01-load-condiments-in-fridge/task.yaml", "catalog_sha256": "2aa9dd2770303990731a430321bda10bf8317f5647c0a70648d1a95692278d91", "current_instruction": "Place the condiments from the counter on the top fridge shelf. Move any existing top-shelf items to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "measured_instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-01-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-01-instruction02", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "Both condiments on the top rack; existing top-rack items on other racks; gripper clear of all four objects.", "finding": "Retain the dynamic food identities and release/retreat clause. The terminal floor placements miss the storage goal itself.", "structural_difficulty": "Shelf rearrangement and collision-aware transport.", "learning_hypothesis": "Reusable routines for shelf rearrangement and collision-aware transport. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/loading_fridge/load_condiments_in_fridge.py", "sha256": "374a8bf1a2a0b7ec8eb6df4c471b162d9c8e156d35d28580afdb6cc574807b3d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 155.9, "step_limit": 20000, "steps": 3118, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 164, "model_tool_calls": 163, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "LoadCondimentsInFridge" }, "backend_task": "robocasa.LoadCondimentsInFridge", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/LoadCondimentsInFridge", "instruction": "Place the mayonnaise and mustard from the counter to the top shelf of the fridge. If the existing items in the fridge are on the top shelf, move them to other shelves. Release the items and withdraw the gripper well clear of all stored items.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-01-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 2, "native_task_id": "robocasa/FilterMicrowavableItem", "catalog_path": "tasks/task01/catalog/02-filter-microwavable-item/task.yaml", "catalog_sha256": "8ca8d7973c981faece4daf827ab92a4380b60c8d127523b4036192376e4f796c", "current_instruction": "Move the non-microwavable food named in the scene instruction from the bowl onto the small plate. Put the bowl containing only the microwavable food in the microwave, close the door, and press start. Release the bowl and withdraw the gripper well clear of it.", "measured_instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-02-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-02-instruction02", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Fruit on the plate, meat in the bowl, bowl in the microwave, door closed, microwave on, gripper clear of bowl.", "finding": "The sampled native labels identify the non-microwavable fruit and microwavable meat. Terminal evidence does not show the required bowl-and-meat containment.", "structural_difficulty": "Selective removal, nested containment, door and switch control.", "learning_hypothesis": "Reusable routines for selective removal, nested containment, door and switch control. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/microwaving_food/filter_microwavable_item.py", "sha256": "89ef510ac1ad0e1fd13094e477a1bed00a2361a351625778fe8cd8289aeab594", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 297.4, "step_limit": 20000, "steps": 5948, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 283, "model_tool_calls": 282, "measured_protocol": { "episode_specs": [ { "instruction_override": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "FilterMicrowavableItem" }, "backend_task": "robocasa.FilterMicrowavableItem", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/FilterMicrowavableItem", "instruction": "Remove the mango from the bowl and place it on the small plate. Then place the bowl with only the steak in the microwave, close the door, and press the start button to microwave the steak. Release the bowl and withdraw the gripper well clear of it.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-02-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 3, "native_task_id": "robocasa/StoreDumplings", "catalog_path": "tasks/task01/catalog/03-store-dumplings/task.yaml", "catalog_sha256": "2bd709f6e17025d223adf615c11d35cd4b7d40f5d1251297de81d469c947f3ae", "current_instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "measured_instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task01-03-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-03-instruction02-providerrecovery01", "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "Exactly two dumplings per container, both containers on fridge racks, gripper clear of food and containers.", "finding": "Keep side-by-side placement: the native receptacle helper requires direct contact, so stacking a dumpling on another can miss it. Containers remain outside the fridge in terminal evidence.", "structural_difficulty": "Counting, packing, and transporting filled containers.", "learning_hypothesis": "Reusable routines for counting, packing, and transporting filled containers. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/storing_leftovers/store_dumplings.py", "sha256": "666a531feb0a6a250953ccb0eebf32b72d763660aecce198b504dc722593ae77", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 830.75, "step_limit": 20000, "steps": 16615, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 142, "model_tool_calls": 141, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "StoreDumplings" }, "backend_task": "robocasa.StoreDumplings", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/StoreDumplings", "instruction": "Place two dumplings side by side in each food container. Put both filled containers on a shelf inside the fridge. Release the moved items and withdraw the gripper well clear of all the food and containers involved.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-03-instruction02-providerrecovery01/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 4, "native_task_id": "robocasa/DivideBuffetTrays", "catalog_path": "tasks/task01/catalog/04-divide-buffet-trays/task.yaml", "catalog_sha256": "ceea57e5a9b3fc54ec2aae8c39284dc20b809b1ae592219956ec619dd8672571", "current_instruction": "Place the two food groups named in the scene instruction on separate dining-counter trays, one group per tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "measured_instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-04-instruction02-providerrecovery01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-04-instruction02-providerrecovery01", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Two sampled food groups on separate trays with no cross-group membership; gripper clear of the food and trays.", "finding": "Group names must remain episode-specific. Food remains in the fridge; the failed result is not just a missing retreat. The initial camera does not show the whole workspace.", "structural_difficulty": "Category allocation and multi-location transport.", "learning_hypothesis": "Reusable routines for category allocation and multi-location transport. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/arranging_buffet/divide_buffet_trays.py", "sha256": "ecd23b0480a93254ea14f70d3141d450479f83df5c093f41a09cdcad90f1b545", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 783.65, "step_limit": 20000, "steps": 15673, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 210, "model_tool_calls": 209, "measured_protocol": { "episode_specs": [ { "instruction_override": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "DivideBuffetTrays" }, "backend_task": "robocasa.DivideBuffetTrays", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/DivideBuffetTrays", "instruction": "Gather the mushroom and bell pepper from the fridge and place them on a tray on the dining counter. Then gather the chicken drumsticks from the fridge and place them on the other tray. Release the food and withdraw the gripper well clear of both trays and all the food.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-04-instruction02-providerrecovery01/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 5, "native_task_id": "robocasa/MakeCheesecakeFilling", "catalog_path": "tasks/task01/catalog/05-make-cheesecake-filling/task.yaml", "catalog_sha256": "e8c06e4f099765d6b9d30641623d5105296398df24184731195bd964238a0ce1", "current_instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "measured_instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task01-05-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-05-instruction02", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Butter, sugar cube and cream cheese in mixer bowl; mixer head lowered; speed dial on.", "finding": "The lowered-head clause is necessary and already concise. The recorded speed dial remains off. No requirement to produce a visibly mixed substance is added.", "structural_difficulty": "Ingredient insertion and sequential appliance operation.", "learning_hypothesis": "Reusable routines for ingredient insertion and sequential appliance operation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/mixing_ingredients/make_cheesecake_filling.py", "sha256": "905f1185f5f72e42bb14ed4e8f8afa282859d2b093e704743c2ff83cdf66b7ad", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 730.05, "step_limit": 20000, "steps": 14601, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 282, "model_tool_calls": 281, "measured_protocol": { "episode_specs": [ { "instruction_override": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "MakeCheesecakeFilling" }, "backend_task": "robocasa.MakeCheesecakeFilling", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/MakeCheesecakeFilling", "instruction": "Put the butter, sugar cube and cream cheese into the mixer bowl, lower the mixer head fully, and turn on the mixer.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-05-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 6, "native_task_id": "robocasa/MultistepSteaming", "catalog_path": "tasks/task01/catalog/06-multistep-steaming/task.yaml", "catalog_sha256": "597068f649a9f1104372843ffd5262c03203afadca9e818208db6f37df505851", "current_instruction": "Turn on the sink faucet and rinse the food item named in the scene instruction in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the specified burner.", "measured_instruction": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-06-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-06-completion01", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Faucet turned on; sampled food in sink while water runs; faucet turned off; same food in pot; pot on sampled burner.", "finding": "Keep 'that same item': the native wording can call a sampled lemon wedge a vegetable. The source does not require turning on the stove or waiting for actual steaming.", "structural_difficulty": "Ordered state changes and transfer of a wet item and pot.", "learning_hypothesis": "Reusable routines for ordered state changes and transfer of a wet item and pot. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/steaming_food/multistep_steaming.py", "sha256": "e745045ff2b19773ca6ff33fd3485daad38266bb772d06ea5f112c3ad8eff94d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 464.4, "step_limit": 20000, "steps": 9288, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 360, "model_tool_calls": 359, "measured_protocol": { "episode_specs": [ { "instruction_override": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "MultistepSteaming" }, "backend_task": "robocasa.MultistepSteaming", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/MultistepSteaming", "instruction": "Turn on the sink faucet and rinse the lemon wedge in the sink under running water. Turn off the faucet, transfer that same item to the pot next to the stove, and place the pot on the rear left burner.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-06-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20.0, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 7, "native_task_id": "robocasa/ScalePortioning", "catalog_path": "tasks/task01/catalog/07-scale-portioning/task.yaml", "catalog_sha256": "9be088861c6eed90e19076ff1c73e78424c66c89827c129e64d4d29d570890ce", "current_instruction": "Place the specified food from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "measured_instruction": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task01-07-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task01-07-completion01", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Food on the scale while gripper is clear for the native reading interval, then food on the dining-counter plate with gripper clear.", "finding": "Retain release, waiting, and final retreat. The native weighing state is latched. The terminal food did not reach the complete weigh-and-plate goal.", "structural_difficulty": "Temporary placement, dwell time, and final relocation.", "learning_hypothesis": "Reusable routines for temporary placement, dwell time, and final relocation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/portioning_meals/scale_portioning.py", "sha256": "50b6fa4bd2415d18ad6cb3e4f9ac7021cfdc47d8720b5700c1a4e97156708579", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 243.9, "step_limit": 20000, "steps": 4878, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 213, "model_tool_calls": 212, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "ScalePortioning" }, "backend_task": "robocasa.ScalePortioning", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/ScalePortioning", "instruction": "Place the chicken drumstick from the fridge on the digital scale. Release it and withdraw the gripper so the food rests freely on the scale, then wait a few seconds for a reading. Move the food onto the dining-counter plate, release it, and withdraw the gripper well clear of the food.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-07-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20.0, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task01", "slot": 8, "native_task_id": "robocasa/ScrubCuttingBoard", "catalog_path": "tasks/task01/catalog/08-scrub-cutting-board/task.yaml", "catalog_sha256": "8fa3be64f47bcd0f4bbe12a43a02cddcfef25ce5cc72820c53f192ab564ec378", "current_instruction": "Pick up the sponge and scrub across different parts of the cutting board with several broad sweeps, keeping the sponge pressed against the board. Put down and release the sponge, then withdraw the gripper well clear of it.", "measured_instruction": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task01-08-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task01-08-instruction02", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "Held sponge contacts the board at multiple separated positions, covers the required extent, then gripper moves clear.", "finding": "Several broad contact sweeps retain the native coverage condition; a single press does not. Remove the repeated explanation of broad coverage. Native success is geometric contact coverage, not simulated dirt removal.", "structural_difficulty": "Tool contact, surface coverage, and release.", "learning_hypothesis": "Reusable routines for tool contact, surface coverage, and release. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/robocasa/robocasa/environments/kitchen/composite/sanitizing_cutting_board/scrub_cutting_board.py", "sha256": "3a4082816ead472ea533bad42fe11a198c8baf10ded05abeef0fbb58e1b1e3e0", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": null, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 75.2, "step_limit": 20000, "steps": 1504, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 46, "model_tool_calls": 45, "measured_protocol": { "episode_specs": [ { "instruction_override": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "max_steps": 20000, "seed": 0, "task": { "backend": "robocasa", "backend_kwargs": { "max_steps": 20000, "stop_on_success": true, "task_name": "ScrubCuttingBoard" }, "backend_task": "robocasa.ScrubCuttingBoard", "budget_mode": "official", "device_bindings": null, "execution_mode": "stepped", "id": "robocasa/ScrubCuttingBoard", "instruction": "Pick up the sponge and clean the cutting board with several broad scrubbing sweeps across different parts of its surface, keeping the sponge pressed against the board while scrubbing. Cover a substantial area of the board rather than pressing in one spot. Put down and release the sponge, then withdraw the gripper well clear of it.", "robot": "robocasa/robocasa-panda-omron", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "robocasa", "version": "0.7.0", "backend": "robocasa", "selection": "all", "protocol": { "instruction_overrides": [ "robocasa/StoreDumplings", "robocasa/MakeCheesecakeFilling", "robocasa/ScrubCuttingBoard" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task01-08-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "recorded-native-integration-states", "command_resimulation": false, "validation_scope": "every recorded native integration state", "intermediate_states_verified": true, "speed": 4, "fps": 20, "source_fps": 20.0, "waiting_policy": "Recorded states retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task02", "slot": 1, "native_task_id": "libero/10_9", "catalog_path": "tasks/task02/catalog/01-mug-in-microwave/task.yaml", "catalog_sha256": "2e4f38c3327a60978fdc1f9bd67a9d41edb7469d2cf3cae4431d6b86fae9ae5d", "current_instruction": "put the yellow and white mug in the microwave and close it", "measured_instruction": "put the yellow and white mug in the microwave and close it", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "t03-kinex-v0.8.0-episode_01", "measured_task_id": 3, "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "In(white_yellow_mug, microwave_heating_region) and Close(microwave).", "finding": "Native BDDL byte hash matches the measured run. The yellow-and-white mug is distinguishable from the other mug. Both original clauses map directly to the goal; keep unchanged.", "structural_difficulty": "Placement into an articulated fixture and door closure.", "learning_hypothesis": "Reusable routines for placement into an articulated fixture and door closure. No RSI gain is measured by this audit.", "source": { "relative_path": "bddl_files/libero_10/KITCHEN_SCENE6_put_the_yellow_and_white_mug_in_the_microwave_and_close_it.bddl", "sha256": "456d145f92be049f445fc77673dc583d9d17ea7afe92f6ffcb2fd1fa5565d420", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Retained pinned BDDL snapshot matching the measured initialization hash" }, "verdict": { "episode": 1, "initialization": { "bddl_sha256": "456d145f92be049f445fc77673dc583d9d17ea7afe92f6ffcb2fd1fa5565d420", "control_frequency_hz": 20, "controller": "JOINT_POSITION/set_qpos", "init_state_index": 0, "init_state_sha256": "6b75c8d4d1e1c40b5d99c78ceea380d99ccffb0a484a4cb2d3153488a0afbbe2", "initialization_frames": 5, "loaded_state_sha256": "ea7db2b727794199cfa1e42f8d924c5841e4af18716a392adf30d5517114c87f", "max_steps": 20000, "seed": 0, "suite": "libero_10", "task_id": 9, "task_order_index": 0 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 266.35, "step_limit": 20000, "steps": 5327, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 103, "model_tool_calls": 112, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "libero", "backend_kwargs": { "init_state_index": 0, "max_steps": 20000, "stop_on_success": true, "suite_name": "libero_10", "task_id": 9 }, "backend_task": "libero.10_9", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "libero/10_9", "instruction": "put the yellow and white mug in the microwave and close it", "robot": "libero/libero-panda-kitchen", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "libero-curated", "version": "0.7.0", "backend": "libero", "selection": "all", "protocol": { "episodes": 3, "seeds": [ 0, 1, 2 ], "initial_state_indices": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/t03/kinex-v0.8.0/episode_01/edited.mp4", "playback": { "command_resimulation": false, "fps": 20, "intermediate_states_verified": null, "kind": "recorded-native-integration-states", "source_fps": 20.0, "speed": 4, "validation_scope": null, "waiting_policy": "Synchronous model waiting omitted; recorded native states retained in HD sources" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task02", "slot": 2, "native_task_id": "libero/10_3", "catalog_path": "tasks/task02/catalog/02-bowl-in-drawer/task.yaml", "catalog_sha256": "529b914ad40d70ecfc9f62b59c4f0567a5c62e7d4a8617c2833b5524e0ea3bfe", "current_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "measured_instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "t04-kinex-v0.8.0-episode_01", "measured_task_id": 4, "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "In(black_bowl, cabinet_bottom_region) and Close(cabinet_bottom_region).", "finding": "Native BDDL byte hash matches the measured run. The black bowl and bottom drawer are identifiable. Keep the original instruction unchanged.", "structural_difficulty": "Drawer access, containment, and closure.", "learning_hypothesis": "Reusable routines for drawer access, containment, and closure. No RSI gain is measured by this audit.", "source": { "relative_path": "bddl_files/libero_10/KITCHEN_SCENE4_put_the_black_bowl_in_the_bottom_drawer_of_the_cabinet_and_close_it.bddl", "sha256": "5255fe54d7f25fad4dee8fa30a30033d8cb908f1708d953c91ab609264fb4fb8", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Retained pinned BDDL snapshot matching the measured initialization hash" }, "verdict": { "episode": 1, "initialization": { "bddl_sha256": "5255fe54d7f25fad4dee8fa30a30033d8cb908f1708d953c91ab609264fb4fb8", "control_frequency_hz": 20, "controller": "JOINT_POSITION/set_qpos", "init_state_index": 0, "init_state_sha256": "e98ded151f64a036c74c872336dd09b968c379f3c9db7bc333aba37e7c235b86", "initialization_frames": 5, "loaded_state_sha256": "ea2ad0a67ceabe915bc4744e8231bf7fe71bddb74ff8c67eae6c821ff82078a4", "max_steps": 20000, "seed": 0, "suite": "libero_10", "task_id": 3, "task_order_index": 0 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 794.1, "step_limit": 20000, "steps": 15882, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 200, "model_tool_calls": 204, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "libero", "backend_kwargs": { "init_state_index": 0, "max_steps": 20000, "stop_on_success": true, "suite_name": "libero_10", "task_id": 3 }, "backend_task": "libero.10_3", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "libero/10_3", "instruction": "put the black bowl in the bottom drawer of the cabinet and close it", "robot": "libero/libero-panda-kitchen", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native control steps", "version": "episodic-learning-v1" }, "current_manifest": { "name": "libero-curated", "version": "0.7.0", "backend": "libero", "selection": "all", "protocol": { "episodes": 3, "seeds": [ 0, 1, 2 ], "initial_state_indices": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/t04/kinex-v0.8.0/episode_01/edited.mp4", "playback": { "command_resimulation": false, "fps": 20, "intermediate_states_verified": null, "kind": "recorded-native-integration-states", "source_fps": 20.0, "speed": 4, "validation_scope": null, "waiting_policy": "Synchronous model waiting omitted; recorded native states retained in HD sources" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 1, "native_task_id": "robotwin/robotwin-aloha.handover_block", "catalog_path": "tasks/task03/catalog/01-handover-block/task.yaml", "catalog_sha256": "28a8cf9167406c4d078635aec14a6bfb6ae76fe929e30a35eaf3d3cb69e30b7d", "current_instruction": "Lift the block with the left arm, pass it to the right arm, then place it upright on the blue pad.", "measured_instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-01-kinex-v0.8.2-episode_01", "measured_task_id": "task03-01", "previous_slot": 1, "recommendation": "Retain with stated limits", "native_conditions": "Block functional point reaches the blue pad pose and the right gripper has an open command.", "finding": "Conditional retention only under the native final-state metric. The predicate has no history of left-to-right transfer; it cannot independently certify the requested handover. Do not weaken the instruction to turn this into plain placement. Gripper commands also do not prove completed physical release.", "structural_difficulty": "Bimanual transfer and upright placement, if the trajectory actually performs the transfer.", "learning_hypothesis": "Reusable routines for bimanual transfer and upright placement, if the trajectory actually performs the transfer. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/handover_block.py", "sha256": "5439712296d7e143d41f8e78b4ab86cd4fad7fb98f40f5f5abb067a462d17650", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 6023, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "cba5c9b6436448e0067283925eaaf21662dcfd9b046e79a5ab3df813763a1a17", "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 11625, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "task_config": "demo_clean", "wall_time_s": 43.96118344925344 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 69.840003317222, "step_limit": 20000, "steps": 1746, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 35, "model_tool_calls": 39, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "handover_block" }, "backend_task": "robotwin-aloha.handover_block", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.handover_block", "instruction": "use the left arm to grasp the red block on the table, handover it to the right arm and place it on the blue pad", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 2, "native_task_id": "robotwin/robotwin-aloha.place_dual_shoes", "catalog_path": "tasks/task03/catalog/02-place-dual-shoes/task.yaml", "catalog_sha256": "58a5a9d9f740fbf97607da25acc7150c8ff1f5224168ade864fe5eb1c1ed29b0", "current_instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes.", "measured_instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-02-instruction02-kinex-v0.9.0-episode_01", "measured_task_id": "task03-02-instruction02", "previous_slot": 2, "recommendation": "Review before final inclusion", "native_conditions": "Both shoe poses near fixed left-front/right-back targets with toes left; both native gripper-open flags true.", "finding": "The native predicate DOES check both gripper-open flags, but those flags read commanded opening. In the measured journal, right-gripper opening is commanded at frame 1852 and success occurs at frame 1853 while the right finger joints remain about 0.0029 m, far below their approximately 0.045 m open pose. Physical release and stable hands-off placement are unproved. Recommend withholding this task from an unconditional final set under the unchanged predicate. It remains selected here because this request synchronizes the existing catalog rather than authorizing a new selection deletion.", "structural_difficulty": "Dual object reorientation and constrained placement.", "learning_hypothesis": "Reusable routines for dual object reorientation and constrained placement. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/place_dual_shoes.py", "sha256": "f35c4a133e1de567b274cbd15f158f585eb05e32aeb05be88839d249d0d7ab9e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 5883, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "a715656d4d3fff3a16238f57b8b2cbe6574b5fc164600c21f74bf7c3f8156513", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 11485, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 43.20029250346124 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 74.12000352051109, "step_limit": 20000, "steps": 1853, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 78, "model_tool_calls": 80, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "max_steps": 20000, "seed": 0, "task": { "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "place_dual_shoes" }, "backend_task": "robotwin-aloha.place_dual_shoes", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.place_dual_shoes", "instruction": "Place both shoes flat inside the smooth bright orange box, toes pointing left. Put the shoe initially on the left in the front half (closer to the robot), and the other shoe in the back half. Release both shoes and withdraw the open grippers.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-02-instruction02/kinex-v0.9.0/episode_01/presentations/legacy-4x-20261002/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 3, "native_task_id": "robotwin/robotwin-aloha.put_bottles_dustbin", "catalog_path": "tasks/task03/catalog/03-put-bottles-dustbin/task.yaml", "catalog_sha256": "1bba9b202b74698e9f71100d36492fcb0751852dd227665000ce14ae546b3edb", "current_instruction": "Move all three bottles from the table into the dustbin on the left.", "measured_instruction": "Move all three bottles from the table into the dustbin on the left. Finish with all three bottles inside the dustbin.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task03-05-kinex-v0.8.2-episode_01", "measured_task_id": "task03-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "All three bottle actor centers inside the fixed dustbin volume.", "finding": "The initial three bottles and left bin match the wording. Terminal views show bottles in the bin. Remove the repeated destination sentence. The native center test is not full-mesh containment, hands-off stability, or a handover test.", "structural_difficulty": "Repeated grasping and disposal of different bottle shapes.", "learning_hypothesis": "Reusable routines for repeated grasping and disposal of different bottle shapes. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/put_bottles_dustbin.py", "sha256": "1c03a4b227daa7d285fa366944f1f75773e32dc6a7b73ab693217a13fcd42795", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": "AssertionError: target_pose cannot be None for move action.", "expert_physics_steps": 16027, "expert_plan_success": true, "expert_prepass": true, "loaded_state_sha256": "1b1ea0b3953b47202b86cd6ac2ffc5cd2dac0d741937943832cae9e93f50a6e7", "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 22829, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "task_config": "demo_clean", "wall_time_s": 117.95161530468613 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 62.960002990439534, "step_limit": 20000, "steps": 1574, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 56, "model_tool_calls": 58, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "put_bottles_dustbin" }, "backend_task": "robotwin-aloha.put_bottles_dustbin", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.put_bottles_dustbin", "instruction": "Move all three bottles from the table into the dustbin on the left. Finish with all three bottles inside the dustbin.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted", "excluded_initialization_diagnostics": [ "expert_error", "expert_plan_success" ], "comparison_tolerance_unchanged": true }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task03", "slot": 4, "native_task_id": "robotwin/robotwin-aloha.scan_object", "catalog_path": "tasks/task03/catalog/04-scan-object/task.yaml", "catalog_sha256": "2ac38bcf96e23a252a278cd9ec0499837bde2cf0fe5aad9312d8e821cf048e66", "current_instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "measured_instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task03-06-instruction03-kinex-v0.9.0-episode_01", "measured_task_id": "task03-06-instruction03", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Both grippers commanded closed; box bottom-center reference close to and in front of scanner functional point, with bounded lateral offset.", "finding": "The instruction identifies the two held objects, the scanner optical window, the center of the box bottom, opposing faces, and an approximate 2\u20133 cm separation. Ten static physical-surface checks across all five scanner models pass the exact native predicate. Under the stated selection criterion, correct execution of this clear goal can pass; the verifier need not reject every incomplete pose. The live checker can trigger before the faces become parallel, so this native success does not independently verify precise face orientation or the literal surface gap. The task retains grasp stability, object-frame reasoning, inverse-kinematics constraints and two-arm alignment. One successful seed establishes a successful native attempt under this protocol; it does not estimate general success rate, prove instruction changes caused the improvement, or establish an RSI learning gain. It should no longer be described as unsolved in this latest run.", "structural_difficulty": "Relative six-dimensional alignment while holding two objects.", "learning_hypothesis": "Reusable routines for relative six-dimensional alignment while holding two objects. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboTwin/envs/scan_object.py", "sha256": "860dfe2cc86d180096089bc87502c7904f7ce2efb64fb32620c1720ca0178c2b", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "attempts": [ { "attempt": 0, "seed": 0, "success": true } ], "control_decimation": 10, "control_frequency_hz": 25, "effective_seed": 0, "expert_error": null, "expert_physics_steps": 3070, "expert_plan_success": false, "expert_prepass": true, "loaded_state_sha256": "6ce2d4764c6abe9d08b1eabef8ced34805aa4dc7bb02848bdbf86fca92112ed6", "max_settling_checks": 1, "max_steps": 20000, "native_timestep_s": 0.004000000189989805, "physics_frequency_hz": 250, "physics_steps": 8672, "protocol": "absolute_joint_position_25hz_practice", "requested_seed": 0, "rig": { "config_sha256": "a4f043d2d4b44cca5ec47fb498a3194da3f87ff9530dcbfdcee1539a44423d57", "urdf_sha256": "097c59fb19a7b482249c6097df8319586ea7cfd268c015516f103b289a7e761a" }, "seed_substitution_allowed": false, "stability_checks": [ [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ], [ { "physics_steps": 2500, "stable": true, "unstable_objects": [] } ] ], "task_config": "demo_clean", "wall_time_s": 71.6042634230107 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 162.4800077173859, "step_limit": 20000, "steps": 4062, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 57, "model_tool_calls": 56, "measured_protocol": { "episode_specs": [ { "instruction_override": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "max_steps": 20000, "seed": 0, "task": { "backend": "robotwin", "backend_kwargs": { "allow_seed_substitution": false, "embodiment": "aloha_agilex", "max_steps": 20000, "stop_on_success": true, "task_config": "demo_clean", "task_name": "scan_object" }, "backend_task": "robotwin-aloha.scan_object", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robotwin/robotwin-aloha.scan_object", "instruction": "Hold the tea box and scanner in separate grippers. Position the scanner's optical reading window directly opposite the center of the box's bottom face, with the two faces facing each other about 2\u20133 cm apart.", "robot": "robotwin/robotwin-aloha-agilex", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task03", "version": "0.8.0", "backend": "robotwin", "selection": "all", "protocol": { "instruction_overrides": [ "robotwin/robotwin-aloha.handover_block", "robotwin/robotwin-aloha.place_dual_shoes", "robotwin/robotwin-aloha.put_bottles_dustbin", "robotwin/robotwin-aloha.scan_object" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task03-06-instruction03/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false, "latest_evidence": "scanner-review.json", "static_positive_alignment_cases": 10 }, { "family": "task04", "slot": 1, "native_task_id": "robodojo/make_toast", "catalog_path": "tasks/task04/catalog/01-robodojo-make-toast/task.yaml", "catalog_sha256": "2385eb2103749e2dd1e940ab2fe7b26683abf3c3de6b7754818b12e3ff7c016b", "current_instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "measured_instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-01-instruction01-kinex-v0.8.2-episode_01", "measured_task_id": "task04-01-instruction01", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "One bread slice in each toaster slot, all slices upright, remaining slices in the rack, lever down, arms home.", "finding": "The four slices, two slots and rack are visible. Keep the count, one-per-slot, rack, lever and arm-return conditions. The task does not evaluate elapsed cooking or toast color.", "structural_difficulty": "Thin-object insertion, articulated actuation, and multi-stage cleanup.", "learning_hypothesis": "Reusable routines for thin-object insertion, articulated actuation, and multi-stage cleanup. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/make_toast.py", "sha256": "1047f484a73d1f92304393879b51f87e33d661b6ad7937198352fe37a0d3233d", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "f58ca00a4fa671677d9292e7b843cd391932e6867abc2f33a8cd8df9a2e0e9e6", "wall_time_s": 66.90659895632416 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 432.04, "step_limit": 20000, "steps": 10801, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 328, "model_tool_calls": 353, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "make_toast" }, "backend_task": "robodojo.make_toast", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/make_toast", "instruction": "Place two bread slices upright in the toaster, one per slot, leaving the other two on the rack. Press the lever down, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-01-instruction01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 2, "native_task_id": "robodojo/classify_objects_by_language", "catalog_path": "tasks/task04/catalog/02-robodojo-classify-objects-by-language/task.yaml", "catalog_sha256": "13cbc16d3666cf3d0e7c4a52dda0e660b1ebe6867a676929a46f80b4df938440", "current_instruction": "Sort all objects into the baskets according to the current goal's category assignments. Place them fully inside, then return both arms to their starting poses.", "measured_instruction": "Put car objects into the left basket, watch objects into the middle basket, and wooden_toy objects into the right basket, then reset the robot arm.", "scene_resolved_instruction": true, "current_wording_matches_measured": false, "episode_id": "task04-02-recovery01-kinex-v0.8.2-episode_01", "measured_task_id": "task04-02-recovery01", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Every object inside its episode-assigned basket, below its upper boundary, and both arms home.", "finding": "Keep the native reset-resolved category-to-basket mapping. A static generic override would erase the assignments. The catalog adds full insertion and arm return without fixing seed-zero categories.", "structural_difficulty": "Language-conditioned categorization and repeated sorting.", "learning_hypothesis": "Reusable routines for language-conditioned categorization and repeated sorting. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/classify_objects_by_language.py", "sha256": "242a8f048f487eb174ef246be78613bbb5f153549ee11a2650412718cf3b36c9", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "cedae218bff456f1033290de4bf91564e04911f4aec0cd573b70c6220a064517", "wall_time_s": 64.84979563206434 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 262.96, "step_limit": 20000, "steps": 6574, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 102, "model_tool_calls": 111, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "classify_objects_by_language" }, "backend_task": "robodojo.classify_objects_by_language", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/classify_objects_by_language", "instruction": "Put car objects into the left basket, watch objects into the middle basket, and wooden_toy objects into the right basket, then reset the robot arm.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-02-recovery01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 3, "native_task_id": "robodojo/store_laptop_and_headphones", "catalog_path": "tasks/task04/catalog/03-robodojo-store-laptop-and-headphones/task.yaml", "catalog_sha256": "7e044ee32208d5ae3e7440ad6894d775f6df0d05a1126ed7424ca6aa3a974ee3", "current_instruction": "Hang the headphones by the middle of their headband, aligned with the stand cradle so both earcups hang evenly below it. Close the laptop fully and seat it upright in its vertical stand. Release both objects and return both arms to their starting poses.", "measured_instruction": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-04-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-04-completion01", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Headband bridge aligned with stand cradle in position and orientation; laptop closed, upright and inserted in its stand; arms home.", "finding": "Keep earcup/headband orientation and fully closed, upright laptop seating. The latest headphone checks pass; laptop upright/stand checks fail. Wording can be shortened without deleting these goals.", "structural_difficulty": "Hanging a shaped object and articulated-object insertion.", "learning_hypothesis": "Reusable routines for hanging a shaped object and articulated-object insertion. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/store_laptop_and_headphones.py", "sha256": "2bee4e31f0594900629fe7f43266e16fecb310b152930ce51524507136809a92", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "d804229232414a2f4446497c48f57cc03621f2552f2c33ddf0705c5b2b3d39c5", "wall_time_s": 52.43044571578503 }, "native_reward": 0.2, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 781.36, "step_limit": 20000, "steps": 19534, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 324, "model_tool_calls": 323, "measured_protocol": { "episode_specs": [ { "instruction_override": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "store_laptop_and_headphones" }, "backend_task": "robodojo.store_laptop_and_headphones", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/store_laptop_and_headphones", "instruction": "Hang the headphones by the middle of their headband on the stand cradle, aligning the headband with the cradle so both earcups hang evenly below it. Close the laptop fully and place it upright and fully seated in its vertical stand. Release the objects and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-04-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 4, "native_task_id": "robodojo/cover_blocks", "catalog_path": "tasks/task04/catalog/04-robodojo-cover-blocks/task.yaml", "catalog_sha256": "9b3734d07c57b98f83b864350fcdf48b2f6ea317f2527f0d38169e9bec993ae4", "current_instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "measured_instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-05-kinex-v0.8.2-episode_01", "measured_task_id": "task04-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Cover left-to-right then uncover red-green-blue in sequence; cups inverted; blocks remain near initial locations; arms home at the end.", "finding": "Native reward checks six ordered stages. The wording already expresses the needed sequence and invariants; leave it unchanged. First and last views alone do not establish the history, so the sequential native verdict is essential.", "structural_difficulty": "Occlusion memory, ordered manipulation, and invariant preservation.", "learning_hypothesis": "Reusable routines for occlusion memory, ordered manipulation, and invariant preservation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/cover_blocks.py", "sha256": "9d14931b16842b93bea8abf9a07991a93a7e663b93c5fd1695108c8b922ab862", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "08430add27614badf9e3ddf928e22b147a66929e1ea16e717721c49aac7af5df", "wall_time_s": 62.40864769835025 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 196.68, "step_limit": 20000, "steps": 4917, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 68, "model_tool_calls": 73, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "cover_blocks" }, "backend_task": "robodojo.cover_blocks", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/cover_blocks", "instruction": "Use the three cups to cover the blocks one at a time, from left to right. Remember the block colors. Once all three are covered, uncover them one at a time in this order: red, green, blue. Keep every cup upside down throughout, leave the blocks in their original positions, and return both arms to their starting poses when finished.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 5, "native_task_id": "robodojo/play_Xylophone", "catalog_path": "tasks/task04/catalog/05-robodojo-play-xylophone/task.yaml", "catalog_sha256": "fb7f696453717db0c04189de680e456cb59d1cc7bd2073c594c7912f51c04f8c", "current_instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "measured_instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task04-06-kinex-v0.8.2-episode_01", "measured_task_id": "task04-06", "previous_slot": 6, "recommendation": "Retain provisionally", "native_conditions": "Mallet strike point reaches the eight key regions in order, with sufficient lift between successive strikes.", "finding": "Keep the original left-to-right strike instruction. No native arm-return requirement exists. Strike detection is geometric, not a force or audio-quality metric.", "structural_difficulty": "Tool use, repeated targeting, and ordered contact.", "learning_hypothesis": "Reusable routines for tool use, repeated targeting, and ordered contact. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/play_Xylophone.py", "sha256": "b7a4b0bc0457bcf55d714ca1867476fc4aabc53084adfc78d3ff5d1437bf7847", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "2c0b65e83b578e4be99357d487da0acecf0bb83c17a4767dacd3a17044313a4d", "wall_time_s": 47.931230374611914 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 120.04, "step_limit": 20000, "steps": 3001, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 49, "model_tool_calls": 48, "measured_protocol": { "episode_specs": [ { "max_steps": 20000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "play_Xylophone" }, "backend_task": "robodojo.play_Xylophone", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/play_Xylophone", "instruction": "Pick up the mallet and strike all xylophone keys from left to right.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-06/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 4.166666666666667, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 6, "native_task_id": "robodojo/store_tools_in_toolbox", "catalog_path": "tasks/task04/catalog/06-robodojo-store-tools-in-toolbox/task.yaml", "catalog_sha256": "f020d7b09324f3f290f30e847333fcac2b8d50393625494c6a71ceabb2857901", "current_instruction": "Place each tool flat in its matching shaped recess, aligned with the outline and fully below the toolbox rim. Release all tools and return both arms to their starting poses.", "measured_instruction": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-07-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-07-completion01", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Each of four tools near and aligned with its matching recess and contained below the toolbox top; arms home.", "finding": "The shaped recesses identify destinations. Native helpers reject the latest slot alignments; hammer and pliers also miss full containment. Keep flat, matched outline, and below-rim seating while deleting repeated containment phrasing.", "structural_difficulty": "Shape matching, orientation control, and tight packing.", "learning_hypothesis": "Reusable routines for shape matching, orientation control, and tight packing. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/store_tools_in_toolbox.py", "sha256": "2969fc3eb171560b3c0bcea62b9f38b8b35f3aecc9bcc7085c3b1cec365b48fc", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "15029891c722a98c9f1b3c92d88dd5883af61f88169c15c35ff4194ea13e4527", "wall_time_s": 59.26869597937912 }, "native_reward": 0.5, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 773.6, "step_limit": 20000, "steps": 19340, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 153, "model_tool_calls": 152, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "store_tools_in_toolbox" }, "backend_task": "robodojo.store_tools_in_toolbox", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/store_tools_in_toolbox", "instruction": "Place each tool flat and fully into its matching shaped recess in the toolbox, aligned with the recess outline and contained below the top rim. Release all tools and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-07-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 7, "native_task_id": "robodojo/insert_tubes", "catalog_path": "tasks/task04/catalog/07-robodojo-insert-tubes/task.yaml", "catalog_sha256": "a552f7fa5e490120a39ed373f0ea68b6e082f690d30929af1e707e77ffdc2912", "current_instruction": "Insert all three tubes upright and deeply into the rack, pointed ends down and capped ends up. Return both arms to their starting poses.", "measured_instruction": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-08-hard02-kinex-v0.9.0-episode_01", "measured_task_id": "task04-08-hard02", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "All three tubes upright, inside the rack, inserted more than 45 mm, and arms home.", "finding": "Pointed ends down and capped ends up match the saved-layout assets. Recorded insertion depths are about 54.75, 54.85 and 56.39 mm. Remove the duplicate seated-in-rack sentence while retaining deep insertion and arm return.", "structural_difficulty": "Reorientation and three constrained insertions.", "learning_hypothesis": "Reusable routines for reorientation and three constrained insertions. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/insert_tubes.py", "sha256": "9a68795ddbd0b816ffd7627e673390ecd83594c0e9d374d4e3abf3435ad439e1", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "7975670facd722b7366c55760a3bef7de735d9e203c6cc915f89108294f5269c", "wall_time_s": 54.14958868827671 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 204.68, "step_limit": 20000, "steps": 5117, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 86, "model_tool_calls": 85, "measured_protocol": { "episode_specs": [ { "instruction_override": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "insert_tubes" }, "backend_task": "robodojo.insert_tubes", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/insert_tubes", "instruction": "Insert all three tubes upright and deeply into the rack, with their pointed ends down and their capped ends up. Leave the tubes seated in the rack, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-08-hard02/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 8, "native_task_id": "robodojo/deposit_coin", "catalog_path": "tasks/task04/catalog/08-robodojo-deposit-coin/task.yaml", "catalog_sha256": "0fd54af41bcc4695b623c5219c1e7942abd08410f80b894cb8c61c97ef9e18b8", "current_instruction": "Pick up the coin from its holder and deposit it through the slot of the coin bank. Let the coin fall fully inside the bank, then return both arms to their starting poses.", "measured_instruction": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-09-candidate01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-09-candidate01", "previous_slot": 9, "recommendation": "Retain provisionally", "native_conditions": "Entire coin contained inside the bank's lower internal region and arms home.", "finding": "Keep the already revised instruction. It correctly requires full entry, without imposing bottom contact or resting. Native score is containment; it does not explicitly record the slot-crossing trajectory. The historical measured text differs from the current revised text.", "structural_difficulty": "Thin-object pickup, regrasping, and slot insertion.", "learning_hypothesis": "Reusable routines for thin-object pickup, regrasping, and slot insertion. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/deposit_coin.py", "sha256": "f9868ef827ec17b8db6c4650cc5d9a9e593d20d00ee822e2212ce229ec01fb61", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "e4cede8a217a89dcace79b57f7ac154415c0ff088aafabc35612ff514e009ec1", "wall_time_s": 60.23394230008125 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 129.84, "step_limit": 20000, "steps": 3246, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 134, "model_tool_calls": 133, "measured_protocol": { "episode_specs": [ { "instruction_override": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "deposit_coin" }, "backend_task": "robodojo.deposit_coin", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/deposit_coin", "instruction": "Pick up the coin from the holder and insert it through the slot of the coin bank. Let it drop completely to the bottom inside the bank, then return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-09-candidate01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 9, "native_task_id": "robodojo/fasten_screws", "catalog_path": "tasks/task04/catalog/09-robodojo-fasten-screws/task.yaml", "catalog_sha256": "e640e9946e1614e178e6410328c8db2292b4596ad9096ed1b85a3bf83b771ad7", "current_instruction": "Fit all three nuts onto the upright bolts of the same color. Leave each nut upright, centered on its bolt, and fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "measured_instruction": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-10-candidate01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-10-candidate01", "previous_slot": 10, "recommendation": "Retain with stated limits", "native_conditions": "Three same-color nut/bolt matches; nuts upright and centered within 1 mm, seated deeper than 9 mm; grippers open and arms home.", "finding": "Retain with the existing observation caveat and explicit prior user selection. This is nut-on-bolt seating, not tightening torque or thread rotation. Red and blue assemblies pass; pink depth is only about 4.497 mm. Historical blue-object camera inconsistency is not repaired by instruction edits. Keep the current corrected instruction unchanged.", "structural_difficulty": "Color matching and precise multi-object assembly.", "learning_hypothesis": "Reusable routines for color matching and precise multi-object assembly. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/fasten_screws.py", "sha256": "2e187a28e33e44efb87baae6463b3ec35264ccd41e35c21a1be44f6361675185", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "48ec8b603e484e26832d63d493d66605a34af0a50a02e261424b1c36559de6cd", "wall_time_s": 61.955387028865516 }, "native_reward": 0.5, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 138.36, "step_limit": 20000, "steps": 3459, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 89, "model_tool_calls": 88, "measured_protocol": { "episode_specs": [ { "instruction_override": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "fasten_screws" }, "backend_task": "robodojo.fasten_screws", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/fasten_screws", "instruction": "Fit each of the three nuts onto the upright bolt of the same color. Keep each nut upright, center its hole on the bolt, and lower it until fully seated. Release all nuts, open both grippers, and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-10-candidate01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 25.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task04", "slot": 10, "native_task_id": "robodojo/play_stacking_toy", "catalog_path": "tasks/task04/catalog/10-robodojo-play-stacking-toy/task.yaml", "catalog_sha256": "cd1ca677defc449254f2489e332021d8749420ea1203ea17ecd3561f32c65a3b", "current_instruction": "Stack all ten toy pieces on the four pegs, grouping matching shapes together: four pieces on the tallest peg, three on the next tallest, two on the next, and one on the shortest. Center each piece precisely on its peg and lower it into the stack, then return both arms to their starting poses.", "measured_instruction": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task04-11-hard01-kinex-v0.9.0-episode_01", "measured_task_id": "task04-11-hard01", "previous_slot": 11, "recommendation": "Retain provisionally", "native_conditions": "Ten pieces in groups of 4/3/2/1 on their designated pegs, anchor and successive-piece centering within 1 mm, sufficient insertion depth, arms home.", "finding": "The saved-layout shape counts and peg heights support the mapping. Preserve all four group sizes, descending height assignment and precise seating; delete redundant restatement that pieces stay on the toy. The native success already requires every group, not merely partial credit.", "structural_difficulty": "Long assembly sequence, shape grouping, recovery, and precise stacking.", "learning_hypothesis": "Reusable routines for long assembly sequence, shape grouping, recovery, and precise stacking. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/RoboDojo/task/RoboDojo/tasks/play_stacking_toy.py", "sha256": "23361329ee7efe330da1f1c241415205bafed2b6c082a913659d868a80d338c3", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "effective_layout": 0, "embodiment_adaptation": false, "layout_config": "arx_x5", "layout_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54 ], "mode": "native_saved_layout", "physics_steps": 800, "requested_seed": 0, "state_sha256": "3fe1cdb26f6ebaa300edb45f1e2bb34d7e8aa2a734019a7c58cb2e1c52fa9ebc", "wall_time_s": 61.61338068265468 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 371.88, "step_limit": 20000, "steps": 9297, "success": true, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 134, "model_tool_calls": 133, "measured_protocol": { "episode_specs": [ { "instruction_override": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "max_steps": 20000, "seed": 0, "task": { "backend": "robodojo", "backend_kwargs": { "env_cfg_type": "arx_x5", "max_steps": 20000, "stop_on_success": true, "task_name": "play_stacking_toy" }, "backend_task": "robodojo.play_stacking_toy", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "robodojo/play_stacking_toy", "instruction": "Stack all ten toy pieces on the four pegs, grouping pieces of the same shape together. Put the group of four on the tallest peg, the group of three on the next tallest, the group of two on the next, and the single piece on the shortest peg. Center each piece precisely on its peg and lower it into the stack. Leave all pieces on the toy and return both arms to their starting poses.", "robot": "robodojo/robodojo-arx-x5", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 20000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "recording": true, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task04", "version": "0.7.0", "backend": "robodojo", "selection": "all", "protocol": { "instruction_overrides": [ "robodojo/make_toast", "robodojo/store_laptop_and_headphones", "robodojo/cover_blocks", "robodojo/store_tools_in_toolbox", "robodojo/insert_tubes", "robodojo/deposit_coin", "robodojo/fasten_screws", "robodojo/play_stacking_toy" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800 }, "environment": { "realtime": false, "max_steps": 20000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task04-11-hard01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 1, "native_task_id": "behavior/assembling_gift_baskets", "catalog_path": "tasks/task05/catalog/01-behavior-assembling-gift-baskets/task.yaml", "catalog_sha256": "720d0bacf36fb2b9e7c1d826c57d3cd9015de46a22817ec78fb27c2f2dd18a49", "current_instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "measured_instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-01-completion02-kinex-v0.9.0-episode_01", "measured_task_id": "task05-01-completion02", "previous_slot": 1, "recommendation": "Retain provisionally", "native_conditions": "One candle, cheese, cookie and bow in each of four baskets, via four BDDL pairing goals.", "finding": "Native instructions and initial arrangement match the 16-object allocation. The run leaves the required distribution unfinished. Keep the native instruction rather than adding grasping recipes.", "structural_difficulty": "Large object count and balanced allocation across four receptacles.", "learning_hypothesis": "Reusable routines for large object count and balanced allocation across four receptacles. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/assembling_gift_baskets/problem0.bddl", "sha256": "6e6c82bae96d3b4efaf3026ed84c2e0ea1aefa57e2f4763c51d2c2fe7b96267e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "assembling_gift_baskets", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_assembling_gift_baskets_instances/house_double_floor_lower_task_assembling_gift_baskets_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "98c28b1e404021338aabc713f9f2bae257c01bf54242e66afb9321b963893cc4", "split": "train" }, "loaded_state_sha256": "53e2563c4b2f5f77f0a5aa75b3ab6885eb71a2b590d087fbb19579a93e91d344", "native_clock_start_s": 0.32500001695007086, "native_context_start_index": 39, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 509.1397671867162 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 552.1666666666666, "step_limit": 50000, "steps": 16565, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 190, "model_tool_calls": 189, "measured_protocol": { "episode_specs": [ { "instruction_override": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "max_steps": 50000, "seed": 0, "task": { "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "assembling_gift_baskets", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.assembling_gift_baskets", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/assembling_gift_baskets", "instruction": "Place one candle, one butter cookie, one piece of Swiss cheese, and one bow from the table into each of the four wicker baskets on the floor in the living room.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-01-completion02/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 30.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 2, "native_task_id": "behavior/can_meat", "catalog_path": "tasks/task05/catalog/02-behavior-can-meat/task.yaml", "catalog_sha256": "4a67bffba57810f502c28a62ff8a89e376616ba96cf8641f77353d7f92c9686c", "current_instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "measured_instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-02-kinex-v0.8.2-episode_01", "measured_task_id": "task05-02", "previous_slot": 2, "recommendation": "Retain provisionally", "native_conditions": "Two cooked bratwursts in each of two closed jars; jars in cabinet; cabinet closed.", "finding": "The sausages are initially cooked; do not add a new cooking goal. The run does not complete the jar packing. Cabinet access and physical grasping are difficulty factors, not evidence of semantic mismatch.", "structural_difficulty": "Nested containment, exact counts, lids, and cabinet closure.", "learning_hypothesis": "Reusable routines for nested containment, exact counts, lids, and cabinet closure. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/can_meat/problem0.bddl", "sha256": "1862f9bb2fdeb8f528b499e4cf6167aa188321032cf57a1f072de13808383505", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "can_meat", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_can_meat_instances/house_single_floor_task_can_meat_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "b9f2b513b53f96ee0f4f3d75594907b2e343d47aaccf8d520bd4197bdea05534", "split": "train" }, "loaded_state_sha256": "b16803dc3264f390b148d5b51291454b27e452b5878dccc7be99df5adf050e1d", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 390.7425012663007 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 181.53333333333333, "step_limit": 50000, "steps": 5446, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 120, "model_tool_calls": 119, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "can_meat", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.can_meat", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/can_meat", "instruction": "Open the kitchen cabinet, take out the two hinged jars, open them, place exactly two cooked bratwursts from the chopping board on the countertop into each jar, then close both jars, put them back inside the cabinet, and close the cabinet.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-02/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 3, "native_task_id": "behavior/moving_boxes_to_storage", "catalog_path": "tasks/task05/catalog/03-behavior-moving-boxes-to-storage/task.yaml", "catalog_sha256": "971459d6d2b9c79707c0a7e74e18c006ecceb0f101d3b326754bf13e4e7fc8c8", "current_instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "measured_instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-03-scene01-kinex-v0.8.2-episode_01", "measured_task_id": "task05-03-scene01", "previous_slot": 3, "recommendation": "Retain provisionally", "native_conditions": "Either box on the garage floor with the other on top of it.", "finding": "The native disjunction allows either stacking order, matching the original instruction. The selected run uses the corrected house_double_floor_lower scene and achieves native success. Do not reuse the superseded scene-mismatch trial.", "structural_difficulty": "Navigation with large carried objects and stacking.", "learning_hypothesis": "Reusable routines for navigation with large carried objects and stacking. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/moving_boxes_to_storage/problem0.bddl", "sha256": "13c91058b0266345eec5a5019b3882f1a1ff794dec5643548b2d42927b700995", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "moving_boxes_to_storage", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_moving_boxes_to_storage_instances/house_double_floor_lower_task_moving_boxes_to_storage_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "cda5caacc8efe2e62d8645de305d8cb6162ed79c8af3c862cfdbf58fd4aa9ac7", "split": "train" }, "loaded_state_sha256": "3bbc1544b2a23a21cb4c171ed06bc2d18339c9960dfd500eb5e407068fa59504", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 404.5288652488962 }, "native_reward": 1.0, "planned_episodes": 1, "reason": "success", "simulation_time_s": 1109.3, "step_limit": 50000, "steps": 33279, "success": true, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 339, "model_tool_calls": 341, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "moving_boxes_to_storage", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.moving_boxes_to_storage", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/moving_boxes_to_storage", "instruction": "Move the two storage containers from the living room to the garage. In the garage, place one container on the floor and stack the other container on top of it (either order is fine).", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-03-scene01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 4, "native_task_id": "behavior/clean_up_your_desk", "catalog_path": "tasks/task05/catalog/04-behavior-clean-up-your-desk/task.yaml", "catalog_sha256": "f0921d8858e28b67c2f33f684835b1dc8475f271707de7074029cf5f42404b19", "current_instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "measured_instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-04-kinex-v0.8.2-episode_01", "measured_task_id": "task05-04", "previous_slot": 4, "recommendation": "Retain provisionally", "native_conditions": "Folders and books inside bookcase; two pens and pencil in pencil box; stapler, pencil box and closed laptop on desk.", "finding": "The object categories and fixtures match the original text. The existing run leaves organization unfinished; chair/access obstruction does not justify rewriting destinations.", "structural_difficulty": "Multi-category cleanup, small-object placement, and articulated laptop handling.", "learning_hypothesis": "Reusable routines for multi-category cleanup, small-object placement, and articulated laptop handling. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/clean_up_your_desk/problem0.bddl", "sha256": "c81e592bf14dd7b2fbbc2a443a5f4989e5d0d038f20c27cc91a4339b3d14318e", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "clean_up_your_desk", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_clean_up_your_desk_instances/house_single_floor_task_clean_up_your_desk_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "a2ad7b92e2cc970ea6aebab530ee8b154cf603c92a487b0e16777e4110d8c202", "split": "train" }, "loaded_state_sha256": "49cbfd7e726d3c2d553c1a75f2c1dff481954da5b2f1c7e79d3d52cb00645201", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 391.4239665251225 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 467.8666666666667, "step_limit": 50000, "steps": 14036, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 288, "model_tool_calls": 304, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "clean_up_your_desk", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.clean_up_your_desk", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/clean_up_your_desk", "instruction": "In the child's room, clean up the desk: put both folders and both paperback books into the bookcase; put the pencil and both pens into the pencil case and leave the case on the desk; take the stapler out of the bookcase and place it on the desk; move the laptop from the bed onto the desk and close it.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-04/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 5, "native_task_id": "behavior/setting_the_fire", "catalog_path": "tasks/task05/catalog/05-behavior-setting-the-fire/task.yaml", "catalog_sha256": "dcab398c37562863d13141b5db4ef14140d63bc97565270506d5d84f67899867", "current_instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "measured_instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-05-kinex-v0.8.2-episode_01", "measured_task_id": "task05-05", "previous_slot": 5, "recommendation": "Retain provisionally", "native_conditions": "Both firewood pieces burning, on the newspaper and inside fireplace; newspaper burning; lighter off.", "finding": "The original wording captures placement, ignition and extinguishing the lighter. Burning wood outside the fireplace does not satisfy the goal. BDDL checks wood-in-fireplace explicitly; newspaper location is partly mediated by the wood-on-paper relation.", "structural_difficulty": "Relational arrangement and coupled material state changes.", "learning_hypothesis": "Reusable routines for relational arrangement and coupled material state changes. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/setting_the_fire/problem0.bddl", "sha256": "d7d62278c4be89fa1199239267784bbcbc87270a5fb64a6a9705ecfff4525c96", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "setting_the_fire", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_lower_task_setting_the_fire_instances/house_double_floor_lower_task_setting_the_fire_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "534427186b9394df338a54b4a7a6004edbf0b7c0aff5ca6213da9123e27f6fcb", "split": "train" }, "loaded_state_sha256": "898740f9b4bc509c34665aa782a1f8fb0c645998eb553e9c6d4799b519517ed6", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_lower", "seed_selects_instance": false, "wall_time_s": 274.6435562046245 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 703.2, "step_limit": 50000, "steps": 21096, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 381, "model_tool_calls": 380, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "setting_the_fire", "max_steps": 50000, "scene_model": "house_double_floor_lower", "stop_on_success": true }, "backend_task": "behavior.setting_the_fire", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/setting_the_fire", "instruction": "In the living room, place the newspaper from the table into the wood fireplace, then put both pieces of firewood from the floor on top of the newspaper. Use the cigar lighter to ignite any one of the items so that the whole pile catches fire, and then turn the lighter off.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-05/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 6, "native_task_id": "behavior/clean_a_trumpet", "catalog_path": "tasks/task05/catalog/06-behavior-clean-a-trumpet/task.yaml", "catalog_sha256": "5d397a071752e95a2cc6cf534184e8e1025642d3f8ef05a144ee05ab18118e19", "current_instruction": "In the bedroom, use the scrub brush from the desk to remove all dust from the cornet (trumpet) on the desk.", "measured_instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "scene_resolved_instruction": false, "current_wording_matches_measured": false, "episode_id": "task05-06-completion01-kinex-v0.9.0-episode_01", "measured_task_id": "task05-06-completion01", "previous_slot": 6, "recommendation": "Retain with stated limits", "native_conditions": "The cornet is not covered with dust; the visual-particle threshold makes this zero attached dust particles.", "finding": "Retain provisionally with visibility uncertainty. Latest complete evidence leaves 18 of 20 particles, so this is not a near-complete threshold-only failure. The older 19-of-20-cleaned interrupted run is not substituted. Keep 'all dust'; remove suggested turning and multi-view procedure. Visibility/reachability of the last particle across instances remains unproved.", "structural_difficulty": "Contact cleaning of a complex curved object and inspection.", "learning_hypothesis": "Reusable routines for contact cleaning of a complex curved object and inspection. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/clean_a_trumpet/problem0.bddl", "sha256": "3d6019ad5d1f0e4e110217bdba0b43bd4ca0757ad9daae6e873ae5cfa8ffdedb", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "clean_a_trumpet", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_double_floor_upper_task_clean_a_trumpet_instances/house_double_floor_upper_task_clean_a_trumpet_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "5395791af3c42e17a2917bf6f213017603434359c07da3df0b18845aff24bc85", "split": "train" }, "loaded_state_sha256": "acc6ed16c6e3706038fce9f5731ba75fa9b99d0c5f330c34ec3ee36e63e5ca1c", "native_clock_start_s": 0.32500001695007086, "native_context_start_index": 39, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_double_floor_upper", "seed_selects_instance": false, "wall_time_s": 372.69310517050326 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 869.9, "step_limit": 50000, "steps": 26097, "success": false, "valid": true, "usage_complete": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 323, "model_tool_calls": 322, "measured_protocol": { "episode_specs": [ { "instruction_override": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "max_steps": 50000, "seed": 0, "task": { "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "clean_a_trumpet", "max_steps": 50000, "scene_model": "house_double_floor_upper", "stop_on_success": true }, "backend_task": "behavior.clean_a_trumpet", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/clean_a_trumpet", "instruction": "In the bedroom, pick up the scrub brush from the desk and scrub the cornet (trumpet) on the desk. Brush across all its dusty surfaces until all dust specks have been removed. Turn or reposition the cornet as needed to expose and brush remaining dusty areas, and inspect it from multiple sides before finishing.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-06-completion01/kinex-v0.9.0/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20.0, "source_fps": 30.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 7, "native_task_id": "behavior/slicing_vegetables", "catalog_path": "tasks/task05/catalog/07-behavior-slicing-vegetables/task.yaml", "catalog_sha256": "30a114db47d6a36eb4c1f97ae560cb0fc2c1dd3dafd0153bf118c1b8be5cdb76", "current_instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "measured_instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-07-kinex-v0.8.2-episode_01", "measured_task_id": "task05-07", "previous_slot": 7, "recommendation": "Retain provisionally", "native_conditions": "Diced systems for all three vegetable types are real, all five original vegetables are no longer real, and refrigerator closed.", "finding": "The original instruction explicitly asks to dice all five vegetables and close the refrigerator. The run fails during access before completing the transformations. Sampled camera frames do not expose the still-enclosed food; scene identities also come from BDDL and loaded-instance evidence.", "structural_difficulty": "Fixture access, tool contact, and object transformation.", "learning_hypothesis": "Reusable routines for fixture access, tool contact, and object transformation. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/slicing_vegetables/problem0.bddl", "sha256": "783060c491bc4ec9cd9ad2bb2e58331ccc895fb4b0f0af418f9e2437135b15d0", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "slicing_vegetables", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_slicing_vegetables_instances/house_single_floor_task_slicing_vegetables_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "9a46a9096e953ae8eadba3bccf58c5658376ea759ea2284723ca55f7821372cd", "split": "train" }, "loaded_state_sha256": "409b3591247242b9b26db777a53c7bcfa6e94f5890e4ec91588699967e8f0870", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 399.51723534055054 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 106.06666666666666, "step_limit": 50000, "steps": 3182, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 79, "model_tool_calls": 78, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "slicing_vegetables", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.slicing_vegetables", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/slicing_vegetables", "instruction": "From the refrigerator in the kitchen, take out the two bell peppers, the two beets, and the zucchini. Then, on either chopping board on the countertop, use the parer to dice all of them so that only diced bell pepper, diced beet, and diced zucchini remain. Make sure the refrigerator is closed when you finish.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-07/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false }, { "family": "task05", "slot": 8, "native_task_id": "behavior/freeze_pies", "catalog_path": "tasks/task05/catalog/08-behavior-freeze-pies/task.yaml", "catalog_sha256": "12b0e87ddd219b430aa1a58a2e379b1ee4fb04543e2da5b65e3d1488d896f4d0", "current_instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "measured_instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "scene_resolved_instruction": false, "current_wording_matches_measured": true, "episode_id": "task05-08-recovery01-kinex-v0.8.2-episode_01", "measured_task_id": "task05-08-recovery01", "previous_slot": 8, "recommendation": "Retain provisionally", "native_conditions": "Each pie in a separate container; both containers in refrigerator; both pies frozen; refrigerator closed.", "finding": "Keep the original packing and freezing goal. Frozen depends on temperature, not mere fridge containment. The terminal pies remain about 23 C and the manipulation goal is incomplete; this does not diagnose a cooling defect. Startup was completed in the selected run.", "structural_difficulty": "Nested containment, navigation, fixture closure, and delayed temperature change.", "learning_hypothesis": "Reusable routines for nested containment, navigation, fixture closure, and delayed temperature change. No RSI gain is measured by this audit.", "source": { "relative_path": "third_party/BEHAVIOR-1K/bddl3/bddl/activity_definitions/freeze_pies/problem0.bddl", "sha256": "78671ab6e34b48033a01ea4a97ba072d1268935fff11eb71bf8a200685261870", "matches_reviewed_source": true, "matches_measured_source": true, "source_origin": "Current native checkout matching the reviewed and measured source" }, "verdict": { "episode": 1, "initialization": { "activity": "freeze_pies", "activity_instance_id": 0, "control_frequency_hz": 30, "grasping_mode": "physical", "instance": { "file": "house_single_floor_task_freeze_pies_instances/house_single_floor_task_freeze_pies_0_0_template-tro_state.json", "settling_physics_steps": 25, "sha256": "7b1aa7d5c7958488798e17b544ae3d4fa97d3e2250f69d66ea93cc2c3e327931", "split": "train" }, "loaded_state_sha256": "031ba3b5f8f6858abb3bc662bf7ffb6fbb9889c9c93ed4a20f1c6948867ad15e", "native_clock_start_s": 0.35000001825392246, "native_context_start_index": 42, "physics_frequency_hz": 120, "requested_seed": 0, "rig": { "urdf_sha256": "b0d76760cbedfbcbe1ddc280380dd5f497bbca380313bc096d7239a2c50dae61" }, "scene_model": "house_single_floor", "seed_selects_instance": false, "wall_time_s": 378.04605627898127 }, "native_reward": 0.0, "planned_episodes": 1, "reason": "stopped", "simulation_time_s": 1238.7, "step_limit": 50000, "steps": 37161, "success": false, "usage_complete": true, "valid": true }, "execution_status": "finished", "usage_complete": true, "model_responses": 451, "model_tool_calls": 465, "measured_protocol": { "episode_specs": [ { "max_steps": 50000, "seed": 0, "task": { "action_abstraction": "devices", "backend": "behavior", "backend_kwargs": { "activity_instance_id": 0, "activity_name": "freeze_pies", "max_steps": 50000, "scene_model": "house_single_floor", "stop_on_success": true }, "backend_task": "behavior.freeze_pies", "budget_mode": "practice", "device_bindings": null, "execution_mode": "stepped", "id": "behavior/freeze_pies", "instruction": "In the kitchen, take the two apple pies from the plates on the countertop, put each pie into a separate tupperware container taken from the cabinet, place both tupperwares inside the refrigerator, close the refrigerator, and leave them until the pies are frozen.", "robot": "behavior/behavior-r1pro", "seed": 0 }, "yaml": "/opt/benchmark/episode_01.yaml" } ], "episodes": 1, "max_steps": 50000, "memory_retention": "retain", "previous_episode_feedback": false, "realtime": false, "safety_timeout_sec": 28800, "seeds": [ 0 ], "step_unit": "native action frames", "version": "episodic-learning-v1" }, "current_manifest": { "name": "task05", "version": "0.5.0", "backend": "behavior", "selection": "all", "protocol": { "instruction_overrides": [ "behavior/clean_a_trumpet" ], "episodes": 3, "seeds": [ 0, 1, 2 ], "safety_timeout_sec": 28800, "activity_instance_ids": [ 0, 1, 2 ] }, "environment": { "realtime": false, "max_steps": 50000, "stop_on_success": true } }, "video": "https://huggingface.co/datasets/RLE-Bench/libero-long-kinex-v0.8.0-astra-medium-seed0-20260927/resolve/main/episodes/task05-08-recovery01/kinex-v0.8.2/episode_01/edited.mp4", "playback": { "kind": "native-command-replay", "command_resimulation": true, "validation_scope": "initial/final state, control targets, budget and verdict", "intermediate_states_verified": false, "speed": 4, "fps": 20, "source_fps": 5.0, "waiting_policy": "Native control frames retained; wall-clock gaps without native steps omitted" }, "native_verdict_changed": false, "new_evaluation_launched": false } ], "current_selection_scope": "The numbered current catalog and measured instruction are separate; prior audit fields describe historical reviews." }