rai-opensource · wmcclinton · Feb 25, 2025 · Mar 12, 2025 · Mar 12, 2025 · Mar 12, 2025
diff --git a/hand_image_failed_detection.png b/hand_image_failed_detection.png
diff --git a/init_search_for_objects_angle0_back_fisheye_image.png b/init_search_for_objects_angle0_back_fisheye_image.png
diff --git a/init_search_for_objects_angle0_frontleft_fisheye_image.png b/init_search_for_objects_angle0_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle0_frontright_fisheye_image.png b/init_search_for_objects_angle0_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle0_hand_color_image.png b/init_search_for_objects_angle0_hand_color_image.png
diff --git a/init_search_for_objects_angle0_left_fisheye_image.png b/init_search_for_objects_angle0_left_fisheye_image.png
diff --git a/init_search_for_objects_angle0_right_fisheye_image.png b/init_search_for_objects_angle0_right_fisheye_image.png
diff --git a/init_search_for_objects_angle1_back_fisheye_image.png b/init_search_for_objects_angle1_back_fisheye_image.png
diff --git a/init_search_for_objects_angle1_frontleft_fisheye_image.png b/init_search_for_objects_angle1_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle1_frontright_fisheye_image.png b/init_search_for_objects_angle1_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle1_hand_color_image.png b/init_search_for_objects_angle1_hand_color_image.png
diff --git a/init_search_for_objects_angle1_left_fisheye_image.png b/init_search_for_objects_angle1_left_fisheye_image.png
diff --git a/init_search_for_objects_angle1_right_fisheye_image.png b/init_search_for_objects_angle1_right_fisheye_image.png
diff --git a/init_search_for_objects_angle2_back_fisheye_image.png b/init_search_for_objects_angle2_back_fisheye_image.png
diff --git a/init_search_for_objects_angle2_frontleft_fisheye_image.png b/init_search_for_objects_angle2_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle2_frontright_fisheye_image.png b/init_search_for_objects_angle2_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle2_hand_color_image.png b/init_search_for_objects_angle2_hand_color_image.png
diff --git a/init_search_for_objects_angle2_left_fisheye_image.png b/init_search_for_objects_angle2_left_fisheye_image.png
diff --git a/init_search_for_objects_angle2_right_fisheye_image.png b/init_search_for_objects_angle2_right_fisheye_image.png
diff --git a/init_search_for_objects_angle3_back_fisheye_image.png b/init_search_for_objects_angle3_back_fisheye_image.png
diff --git a/init_search_for_objects_angle3_frontleft_fisheye_image.png b/init_search_for_objects_angle3_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle3_frontright_fisheye_image.png b/init_search_for_objects_angle3_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle3_hand_color_image.png b/init_search_for_objects_angle3_hand_color_image.png
diff --git a/init_search_for_objects_angle3_left_fisheye_image.png b/init_search_for_objects_angle3_left_fisheye_image.png
diff --git a/init_search_for_objects_angle3_right_fisheye_image.png b/init_search_for_objects_angle3_right_fisheye_image.png
diff --git a/init_search_for_objects_angle4_back_fisheye_image.png b/init_search_for_objects_angle4_back_fisheye_image.png
diff --git a/init_search_for_objects_angle4_frontleft_fisheye_image.png b/init_search_for_objects_angle4_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle4_frontright_fisheye_image.png b/init_search_for_objects_angle4_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle4_hand_color_image.png b/init_search_for_objects_angle4_hand_color_image.png
diff --git a/init_search_for_objects_angle4_left_fisheye_image.png b/init_search_for_objects_angle4_left_fisheye_image.png
diff --git a/init_search_for_objects_angle4_right_fisheye_image.png b/init_search_for_objects_angle4_right_fisheye_image.png
diff --git a/init_search_for_objects_angle5_back_fisheye_image.png b/init_search_for_objects_angle5_back_fisheye_image.png
diff --git a/init_search_for_objects_angle5_frontleft_fisheye_image.png b/init_search_for_objects_angle5_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle5_frontright_fisheye_image.png b/init_search_for_objects_angle5_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle5_hand_color_image.png b/init_search_for_objects_angle5_hand_color_image.png
diff --git a/init_search_for_objects_angle5_left_fisheye_image.png b/init_search_for_objects_angle5_left_fisheye_image.png
diff --git a/init_search_for_objects_angle5_right_fisheye_image.png b/init_search_for_objects_angle5_right_fisheye_image.png
diff --git a/init_search_for_objects_angle6_back_fisheye_image.png b/init_search_for_objects_angle6_back_fisheye_image.png
diff --git a/init_search_for_objects_angle6_frontleft_fisheye_image.png b/init_search_for_objects_angle6_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle6_frontright_fisheye_image.png b/init_search_for_objects_angle6_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle6_hand_color_image.png b/init_search_for_objects_angle6_hand_color_image.png
diff --git a/init_search_for_objects_angle6_left_fisheye_image.png b/init_search_for_objects_angle6_left_fisheye_image.png
diff --git a/init_search_for_objects_angle6_right_fisheye_image.png b/init_search_for_objects_angle6_right_fisheye_image.png
diff --git a/init_search_for_objects_angle7_back_fisheye_image.png b/init_search_for_objects_angle7_back_fisheye_image.png
diff --git a/init_search_for_objects_angle7_frontleft_fisheye_image.png b/init_search_for_objects_angle7_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle7_frontright_fisheye_image.png b/init_search_for_objects_angle7_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle7_hand_color_image.png b/init_search_for_objects_angle7_hand_color_image.png
diff --git a/init_search_for_objects_angle7_left_fisheye_image.png b/init_search_for_objects_angle7_left_fisheye_image.png
diff --git a/init_search_for_objects_angle7_right_fisheye_image.png b/init_search_for_objects_angle7_right_fisheye_image.png
diff --git a/init_search_for_objects_angle8_back_fisheye_image.png b/init_search_for_objects_angle8_back_fisheye_image.png
diff --git a/init_search_for_objects_angle8_frontleft_fisheye_image.png b/init_search_for_objects_angle8_frontleft_fisheye_image.png
diff --git a/init_search_for_objects_angle8_frontright_fisheye_image.png b/init_search_for_objects_angle8_frontright_fisheye_image.png
diff --git a/init_search_for_objects_angle8_hand_color_image.png b/init_search_for_objects_angle8_hand_color_image.png
diff --git a/init_search_for_objects_angle8_left_fisheye_image.png b/init_search_for_objects_angle8_left_fisheye_image.png
diff --git a/init_search_for_objects_angle8_right_fisheye_image.png b/init_search_for_objects_angle8_right_fisheye_image.png
diff --git a/predicators/approaches/bilevel_planning_approach.py b/predicators/approaches/bilevel_planning_approach.py
@@ -72,6 +72,11 @@ def _solve(self, task: Task, timeout: int) -> Callable[[State], Action]:
             for act in nsrt_plan:
                 logging.debug(act)
 
+            for act in nsrt_plan:
+                print(act)
+            import ipdb
+            ipdb.set_trace()
+
         # Run full bilevel planning.
         else:
             option_plan, nsrt_plan, metrics = self._run_sesame_plan(

diff --git a/predicators/approaches/documentation/grammar_search_invention_approach.md b/predicators/approaches/documentation/grammar_search_invention_approach.md
@@ -62,7 +62,7 @@ apple_coring__vlm_demos__456__2
     | options.txt
 
 ### Running predicate invention using these image demos
-To use the Gemini VLM, you need to set the `GOOGLE_API_KEY` environment variable in your terminal. You can make/get an API key [here](https://aistudio.google.com/app/apikey).
+To use the Gemini VLM, you need to set the `GOOGLE_API_KEY` environment variable in your terminal. You can make/get an API key [here](https://aistudio.google.com/app/apikey). To use an Open AI VLM, the `OPENAI_API_KEY` variable needs to be set.
 
 Example command: `python predicators/main.py --env apple_coring --seed 456 --approach grammar_search_invention --excluded_predicates all --num_train_tasks 1 --num_test_tasks 0 --offline_data_method saved_vlm_img_demos_folder --vlm_trajs_folder_name apple_coring__vlm_demos__456__1`
 

diff --git a/predicators/approaches/grammar_search_invention_approach.py b/predicators/approaches/grammar_search_invention_approach.py
@@ -1021,9 +1021,8 @@ def _parse_atom_dataset_from_annotated_dataset(
                 for ground_atom in ground_atom_state:
                     assert isinstance(ground_atom, GroundAtom)
                     if ground_atom.predicate not in candidates:
-                        # The cost of this predicate is simply its arity.
-                        candidates[ground_atom.predicate] = float(
-                            len(ground_atom.objects))
+                            candidates[ground_atom.predicate] = float(
+                                len(ground_atom.objects))
         logging.debug(f"All candidate predicates: {candidates.keys()}")
         return (atom_dataset, candidates)
 

diff --git a/predicators/approaches/spot_wrapper_approach.py b/predicators/approaches/spot_wrapper_approach.py
@@ -18,6 +18,7 @@
 from predicators import utils
 from predicators.approaches import BaseApproach, BaseApproachWrapper
 from predicators.envs.spot_env import get_detection_id_for_object, get_robot
+from predicators.settings import CFG
 from predicators.spot_utils.skills.spot_find_objects import find_objects
 from predicators.spot_utils.skills.spot_stow_arm import stow_arm
 from predicators.spot_utils.utils import get_allowed_map_regions
@@ -66,23 +67,24 @@ def _policy(state: State) -> Action:
                     state.get(obj, "lost") > 0.5:
                     lost_objects.add(obj)
             # Need to find the objects.
-            if lost_objects:
-                logging.info(f"[Spot Wrapper] Lost objects: {lost_objects}")
-                # Reset the base approach policy.
-                base_approach_policy = None
-                need_stow = True
-                self._base_approach_has_control = False
-                robot, localizer, lease_client = get_robot()
-                lost_object_ids = {
-                    get_detection_id_for_object(o)
-                    for o in lost_objects
-                }
-                allowed_regions = self._allowed_regions
-                extra_info = SpotActionExtraInfo(
-                    "find-objects", [], find_objects,
-                    (state, self._rng, robot, localizer, lease_client,
-                     lost_object_ids, allowed_regions), None, tuple())
-                return utils.create_spot_env_action(extra_info)
+            # NOTE: HACK: commenting out for now - just for some robot testing!
+            # if lost_objects and len(CFG.spot_vlm_teleop_demo_folderpath) == 0:
+            #     logging.info(f"[Spot Wrapper] Lost objects: {lost_objects}")
+            #     # Reset the base approach policy.
+            #     base_approach_policy = None
+            #     need_stow = True
+            #     self._base_approach_has_control = False
+            #     robot, localizer, lease_client = get_robot()
+            #     lost_object_ids = {
+            #         get_detection_id_for_object(o)
+            #         for o in lost_objects
+            #     }
+            #     allowed_regions = self._allowed_regions
+            #     extra_info = SpotActionExtraInfo(
+            #         "find-objects", [], find_objects,
+            #         (state, self._rng, robot, localizer, lease_client,
+            #          lost_object_ids, allowed_regions), None, tuple())
+            #     return utils.create_spot_env_action(extra_info)
             # Found the objects. Stow the arm before replanning.
             if need_stow:
                 logging.info("[Spot Wrapper] Lost objects found, stowing.")

diff --git a/predicators/approaches/vlm_open_loop_approach.py b/predicators/approaches/vlm_open_loop_approach.py
@@ -67,6 +67,39 @@ def get_name(cls) -> str:
     def is_learning_based(self) -> bool:
         return True
 
+    def get_goals_for_specific_datasets(self, train_task_idx: int) -> str:
+        """HACK to get the goals for specific datasets.
+
+        Used for spot envs because the invention env is different from
+        the actual execution env.
+        """
+        if CFG.vlm_trajs_folder_name == "spot_vlm_table_wiping_execution_env__vlm_demos__3__8":
+            if train_task_idx in [0, 2, 3]:
+                return "TableWiped(child_play_table:table)"
+            elif train_task_idx in [1, 4]:
+                return "VLMIn(apple:movable, seethru_plastic_dustbin:movable), TableWiped(child_play_table:table)"
+            elif train_task_idx == 5:
+                return "VLMIn(green_block: movable, cardboard_recycling_bin:movable)"
+            elif train_task_idx == 6:
+                return "VLMIn(orange_block: movable, cardboard_recycling_bin:movable)"
+            elif train_task_idx == 7:
+                return "VLMIn(spam_tin:movable, cardboard_recycling_bin:movable)"
+            else:
+                raise NotImplementedError(
+                    "Shouldn't be getting here! i = {}".format(i))
+        elif CFG.vlm_trajs_folder_name == "spot_vlm_table_wiping_human_execution_env__vlm_demos__3__6":
+            if train_task_idx in [0, 1, 2]:
+                return "TableWiped(child_play_table:table)"
+            elif train_task_idx in [3]:
+                return "VLMIn(apple:movable, seethru_plastic_dustbin:movable), TableWiped(child_play_table:table)"
+            elif train_task_idx == 4:
+                return "VLMIn(green_block: movable, cardboard_recycling_bin:movable)"
+            elif train_task_idx == 5:
+                return "VLMIn(spam_tin:movable, cardboard_recycling_bin:movable)"
+            else:
+                raise NotImplementedError(
+                    "Shouldn't be getting here! i = {}".format(i))
+
     def learn_from_offline_dataset(self, dataset: Dataset) -> None:
         """Adds the images and plans from the training dataset to the base
         prompt for use at test time!"""
@@ -77,7 +110,7 @@ def _append_to_prompt_state_imgs_list(state: State) -> None:
             for img_num, img in enumerate(state.simulator_state["images"]):
                 pil_img = PIL.Image.fromarray(img)  # type: ignore
                 width, height = pil_img.size
-                font_size = 15
+                font_size = 6
                 text = f"Demonstration {traj_num}, " + \
                     f"State {state_num}, Image {img_num}"
                 draw = ImageDraw.Draw(pil_img)
@@ -119,9 +152,16 @@ def _append_to_prompt_state_imgs_list(state: State) -> None:
             segment_traj, ll_traj = seg_traj
             if not ll_traj.is_demo:
                 continue
-            traj_goal = self._train_tasks[ll_traj.train_task_idx].goal
+
+            # Get the goal string depending on the env.
+            if "spot" not in CFG.env:
+                traj_goal = self._train_tasks[ll_traj.train_task_idx].goal
+                traj_goal_str = str(sorted(traj_goal))
+            else:
+                traj_goal_str = self.get_goals_for_specific_datasets(traj_num)
+
             self._prompt_demos_str += f"Demonstration {traj_num}, " + \
-                f"Goal: {str(sorted(traj_goal))}\n"
+                f"Goal: {traj_goal_str}\n"
             assert len(segment_traj) > 0
             for state_num, seg in enumerate(segment_traj):
                 state = seg.states[0]
@@ -162,14 +202,20 @@ def _query_vlm_for_option_plan(self, task: Task) -> Sequence[_Option]:
         assert isinstance(init_state.simulator_state["images"], List)
         curr_options = sorted(self._initial_options)
         imgs = init_state.simulator_state["images"]
-        pil_imgs = [
-            PIL.Image.fromarray(img_arr)  # type: ignore
-            for img_arr in imgs
-        ]
+        if isinstance(imgs[0], np.ndarray):
+            pil_imgs = [
+                PIL.Image.fromarray(img_arr)  # type: ignore
+                for img_arr in imgs
+            ]
+        elif isinstance(imgs[0], PIL.Image.Image):
+            pil_imgs = imgs
+        else:
+            raise ValueError(
+                "Simulator state images are not in a recognized format!")
         imgs_for_vlm = []
         for img_num, pil_img in enumerate(pil_imgs):
             draw = ImageDraw.Draw(pil_img)
-            img_font = utils.get_scaled_default_font(draw, 10)
+            img_font = utils.get_scaled_default_font(draw, 6)
             img_with_txt = utils.add_text_to_draw_img(
                 draw, (50, 50), f"Initial state to plan from, Image {img_num}",
                 img_font)

diff --git a/predicators/datasets/__init__.py b/predicators/datasets/__init__.py
@@ -9,7 +9,8 @@
 from predicators.datasets.generate_atom_trajs_with_vlm import \
     create_ground_atom_data_from_generated_demos, \
     create_ground_atom_data_from_labelled_txt, \
-    create_ground_atom_data_from_saved_img_trajs
+    create_ground_atom_data_from_saved_img_trajs, \
+    create_low_level_trajs_from_saved_img_trajs
 from predicators.datasets.ground_atom_data import create_ground_atom_data
 from predicators.envs import BaseEnv
 from predicators.settings import CFG
@@ -77,7 +78,7 @@ def create_dataset(env: BaseEnv, train_tasks: List[Task],
             "demo+labelled_atoms", "geo_and_demo+labelled_atoms"
     ]:
         return create_ground_atom_data_from_labelled_txt(
-            env, train_tasks, known_options)
+            env, train_tasks, known_options, known_predicates=known_predicates)
     if CFG.offline_data_method in [
             "saved_vlm_img_demos_folder", "geo_and_saved_vlm_img_demos_folder"
     ]:  # pragma: no cover.
@@ -87,6 +88,9 @@ def create_dataset(env: BaseEnv, train_tasks: List[Task],
         # we want to instantiate our own 'dummy' VLM.
         return create_ground_atom_data_from_saved_img_trajs(
             env, train_tasks, known_predicates, known_options)
+    if CFG.offline_data_method == "saved_vlm_img_demos_folder_nolabel":
+        return create_low_level_trajs_from_saved_img_trajs(
+            env, train_tasks, known_predicates, known_options)
     if CFG.offline_data_method == "empty":
         return Dataset([])
     raise NotImplementedError("Unrecognized dataset method.")