From 7b4ed59822da2ed760a5d72564e0712a66b4dea9 Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Thu, 15 Dec 2022 20:01:28 +0100 Subject: [PATCH 01/16] file managment --- .hydra/config.yaml | 36 ++++----- .hydra/hydra.yaml | 17 ++-- .hydra/overrides.yaml | 9 +-- run.py | 80 ++++++++++++------- src/bbox_3D_estimation/main.py | 2 +- src/bbox_3D_estimation/utils.py | 15 ++-- .../detection_2D_utils.py | 7 +- {scripts => src/utils}/parse_scanned_data.py | 61 +++++++++++++- 8 files changed, 153 insertions(+), 74 deletions(-) rename {scripts => src/utils}/parse_scanned_data.py (83%) diff --git a/.hydra/config.yaml b/.hydra/config.yaml index 73f0cd9..5163ec6 100644 --- a/.hydra/config.yaml +++ b/.hydra/config.yaml @@ -1,27 +1,21 @@ work_dir: ${hydra:runtime.cwd} data_dir: ${work_dir}/data print_config: true -type: inference -task_name: demo -num_leaf: 8 -suffix: '' -save_demo: false -save_wis3d: false -use_tracking: false -model: - onepose_model_path: ${work_dir}/data/models/checkpoints/onepose/GATsSPG.ckpt - extractor_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth - match_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth -scan_data_dir: ${data_dir}/onepose_datasets/val_data -sfm_model_dir: ${data_dir}/sfm_model +type: sfm +redo: true +dataset: + max_num_kp3d: 2500 + max_num_kp2d: 1000 + data_dir: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger + tiger-annotate + outputs_dir: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger network: detection: superpoint + detection_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth matching: superglue -max_num_kp3d: 2500 -input: - data_dirs: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-test - sfm_model_dirs: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger -output: - vis_dir: ${work_dir}/runs/vis/demo - eval_dir: ${work_dir}/runs/eval/demo + matching_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth +sfm: + down_ratio: 5 + covis_num: 10 + rotation_thresh: 50 +disable_lightning_logs: true diff --git a/.hydra/hydra.yaml b/.hydra/hydra.yaml index 3f952ed..409e264 100644 --- a/.hydra/hydra.yaml +++ b/.hydra/hydra.yaml @@ -109,15 +109,14 @@ hydra: overrides: hydra: [] task: - - +experiment=test_demo - - use_tracking=False - - input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-test - - input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger + - +preprocess=sfm_spp_spg_demo + - dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger + tiger-annotate + - dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger job: - name: inference_demo - override_dirname: +experiment=test_demo,input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-test,input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger,use_tracking=False + name: run + override_dirname: +preprocess=sfm_spp_spg_demo,dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger + tiger-annotate,dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger id: ??? num: ??? config_name: config.yaml @@ -142,7 +141,7 @@ hydra: schema: structured provider: schema choices: - experiment: test_demo + preprocess: sfm_spp_spg_demo logger: null callbacks: null datamodule: null diff --git a/.hydra/overrides.yaml b/.hydra/overrides.yaml index 31687cf..f309db8 100644 --- a/.hydra/overrides.yaml +++ b/.hydra/overrides.yaml @@ -1,5 +1,4 @@ -- +experiment=test_demo -- use_tracking=False -- input.data_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-test -- input.sfm_model_dirs=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger +- +preprocess=sfm_spp_spg_demo +- dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger + tiger-annotate +- dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger diff --git a/run.py b/run.py index 8b39ef2..5fe4829 100644 --- a/run.py +++ b/run.py @@ -2,6 +2,7 @@ import os import glob import hydra +import numpy as np import os.path as osp from loguru import logger @@ -9,8 +10,8 @@ from omegaconf import DictConfig from src.bbox_3D_estimation.utils import predict_3D_bboxes - -from src.bbox_3D_estimation.utils import predict_3D_bboxes +from src.utils.parse_scanned_data import parse_images +from src.utils import data_utils def merge_( @@ -109,6 +110,7 @@ def merge_anno(cfg): def sfm(cfg): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir + #cfg.sfm.down_ratio = 1 down_ratio = cfg.sfm.down_ratio data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs @@ -117,42 +119,62 @@ def sfm(cfg): root_dir, sub_dirs = data_dir.split(" ")[0], data_dir.split(" ")[1:] # Parse image, intrinsics and poses directories: - img_paths, poses_paths, full_res_img_paths = [], [], [] + poses_paths, full_res_img_paths = [], [] + paths = {} for sub_dir in sub_dirs: - seq_dir = osp.join(root_dir, sub_dir) - img_paths += glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True) - full_res_img_paths += glob.glob(str(Path(seq_dir)) + "/color_full/*.png", recursive=True) - poses_paths += glob.glob(str(Path(seq_dir)) + "/poses/*.txt", recursive=True) - intrinsics_path = str(Path(seq_dir)) + "/intrinsics.txt" - poses_paths += glob.glob(str(Path(seq_dir)) + "/poses/*.txt", recursive=True) - intrinsics_path = str(Path(seq_dir)) + "/intrinsics.txt" + seq_dir = str(Path(osp.join(root_dir, sub_dir))) - # Choose less images from the list to build the sfm model + full_res_img_paths += glob.glob(seq_dir + "/color_full/*.png", recursive=True) + poses_paths += glob.glob(seq_dir + "/poses/*.txt", recursive=True) + intrinsics_path = seq_dir + "/intrinsics.txt" - down_img_lists = [] - for img_file in img_paths: - index = int(img_file.split("/")[-1].split(".")[0]) - if index % down_ratio == 0: - down_img_lists.append(img_file) + paths['final_intrin_file'] = intrinsics_path + paths['reproj_box_dir'] = seq_dir + "/reproj_box/" + paths['intrin_dir'] = seq_dir + "/intrin/" + paths['img_list'] = full_res_img_paths + paths['M_dir'] = seq_dir + "/modified_poses/" + paths['crop_img_root'] = seq_dir + "/color/" - if len(img_paths) == 0: - logger.info(f"No png image in {root_dir}") - continue obj_name = root_dir.split("/")[-1] outputs_dir_root = cfg.dataset.outputs_dir.format(obj_name) # Begin predict 3d bboxes - predict_3D_bboxes( - intrisics_path=intrinsics_path, - full_res_img_paths=full_res_img_paths, - poses_paths=poses_paths, - data_root=root_dir, - seq_dir = seq_dir, - compute_on_GPU="cuda", - step=1, - hololens=cfg.hololens - ) + if not os.path.exists(root_dir + "/box3d_corners.txt"): + predict_3D_bboxes( + intrinsics_path=intrinsics_path, + full_res_img_paths=full_res_img_paths, + poses_paths=poses_paths, + data_root=root_dir, + device="cpu", + root_2d_bbox=paths['reproj_box_dir'], + step=1, + hololens=cfg.hololens, + ) + + # Crop images and save them if color folder is empty + crop_images=True + if crop_images: + img_paths = parse_images(paths, downsample_rate=1, hw=512) + else: + img_paths = [] + for sub_dir in sub_dirs: + img_paths += glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True) + K, _ = data_utils.get_K(intrinsics_path) + for index, _ in enumerate(img_paths): + np.savetxt(paths['intrin_dir'] + f"{index}.txt", K) + + if len(img_paths) == 0: + logger.info(f"No png image in {root_dir}") + continue + + # Choose less images from the list to build the sfm model + + down_img_lists = [] + for img_file in img_paths: + index = int(img_file.split("/")[-1].split(".")[0]) + if index % down_ratio == 0: + down_img_lists.append(img_file) # Begin SfM and postprocess: sfm_core(cfg, down_img_lists, outputs_dir_root) diff --git a/src/bbox_3D_estimation/main.py b/src/bbox_3D_estimation/main.py index 54ea11a..897193b 100644 --- a/src/bbox_3D_estimation/main.py +++ b/src/bbox_3D_estimation/main.py @@ -32,6 +32,6 @@ BboxPredictor=BboxPredictor, full_res_img_paths=img_lists, poses_paths=poses_list, - intrisics_path=K, + intrinsics_path=K, data_root=data_root, ) diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 196aa3c..71b55ba 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -108,21 +108,22 @@ def save_dimensions(self, data_root): def predict_3D_bboxes( full_res_img_paths, - intrisics_path, + intrinsics_path, poses_paths, data_root, seq_dir, step=1, downscale_factor=0.3, - compute_on_GPU="cpu", + device="cpu", + root_2d_bbox=None, hololens=False ): full_res_img_paths = sort_path_list(full_res_img_paths) poses_paths = sort_path_list(poses_paths) - _K, _ = data_utils.get_K(intrisics_path) + _K, _ = data_utils.get_K(intrinsics_path) DetectorBox3D = Detector3D(_K) - BboxPredictor = UnsupBbox(downscale_factor=downscale_factor, device=compute_on_GPU) + BboxPredictor = UnsupBbox(downscale_factor=downscale_factor, device=device) for id, img_path in enumerate(tqdm(full_res_img_paths)): if id % step == 0 or id == 0: @@ -132,7 +133,11 @@ def predict_3D_bboxes( poses_orig = read_list_poses_orig([poses_paths[id]]) bbox_orig_res = BboxPredictor.infer_2d_bbox(image=image, K=_K) - DetectorBox3D.add_view(bbox_orig_res, poses, poses_orig) + if root_2d_bbox is not None: + if not os.path.exists(root_2d_bbox): + os.makedirs(root_2d_bbox) + BboxPredictor.save_2d_bbox(file_path = root_2d_bbox + f"{id}.txt") + DetectorBox3D.add_view(bbox_orig_res, poses) DetectorBox3D.detect_3D_box() DetectorBox3D.save_3D_box(data_root) diff --git a/src/deep_spectral_method/detection_2D_utils.py b/src/deep_spectral_method/detection_2D_utils.py index 3a11de4..cc5426c 100644 --- a/src/deep_spectral_method/detection_2D_utils.py +++ b/src/deep_spectral_method/detection_2D_utils.py @@ -63,7 +63,10 @@ def infer_2d_bbox(self, image, K): # Bounding boxes bbox = extract.extract_bboxes(feature_dict=feature_dict, segmap=segmap) - bbox_orig_res = ( + self.bbox_orig_res = ( np.array(bbox["bboxes_original_resolution"][0]) / self.downscale_factor ) - return bbox_orig_res + return self.bbox_orig_res + + def save_2d_bbox(self, file_path): + np.savetxt(file_path, self.bbox_orig_res, delimiter=" ") diff --git a/scripts/parse_scanned_data.py b/src/utils/parse_scanned_data.py similarity index 83% rename from scripts/parse_scanned_data.py rename to src/utils/parse_scanned_data.py index 653e2fa..bc6a5ea 100644 --- a/scripts/parse_scanned_data.py +++ b/src/utils/parse_scanned_data.py @@ -5,8 +5,10 @@ import os.path as osp import argparse from pathlib import Path +from tqdm import tqdm from transforms3d import affines, quaternions from src.utils import data_utils +from src.bbox_3D_estimation.utils import sort_path_list def get_arkit_default_path(data_dir, AR_annotations=True): @@ -188,10 +190,58 @@ def parse_video(paths, downsample_rate=5, bbox_3d_homo=None, hw=512, AR_annotati index += 1 cap.release() +def parse_images(paths, downsample_rate=5, hw=512): + orig_intrin_file = paths['final_intrin_file'] + crop_img_root = paths['crop_img_root'] + intrin_dir = paths['intrin_dir'] + img_list = sort_path_list(paths['img_list']) -def data_process_anno(data_dir, downsample_rate=1, hw=512, AR_annotations=True): + for key in paths.keys(): + if not isinstance(paths[key], list) and not os.path.exists(paths[key]): + os.makedirs(paths[key]) - paths = get_arkit_default_path(data_dir, AR_annotations) + K, _ = data_utils.get_K(orig_intrin_file) + crop_img_paths = [] + for index, img_path in enumerate(tqdm(img_list)): + image = cv2.imread(str(img_path)) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + if index % downsample_rate != 0: + continue + + save_intrin_path = osp.join(intrin_dir, '{}.txt'.format(index)) + reproj_box3d_file = osp.join(paths['reproj_box_dir'], '{}.txt'.format(index)) + + if not osp.isfile(reproj_box3d_file): + continue + + reproj_box3d = np.loadtxt(osp.join(paths['reproj_box_dir'], '{}.txt'.format(index))).astype(int) + x0, y0 = reproj_box3d[0], reproj_box3d[1] + x1, y1 = reproj_box3d[2], reproj_box3d[3] + + box = np.array([x0, y0, x1, y1]) + resize_shape = np.array([y1 - y0, x1 - x0]) + K_crop, K_crop_homo = data_utils.get_K_crop_resize(box, K, resize_shape) + image_crop, trans1 = data_utils.get_image_crop_resize(image, box, resize_shape) + + box_new = np.array([0, 0, x1-x0, y1-y0]) + resize_shape = np.array([hw, hw]) + K_crop, K_crop_homo = data_utils.get_K_crop_resize(box_new, K_crop, resize_shape) + image_crop, trans2 = data_utils.get_image_crop_resize(image_crop, box_new, resize_shape) + + trans_full_to_crop = trans2 @ trans1 + trans_crop_to_full = np.linalg.inv(trans_full_to_crop) + + crop_img_file = crop_img_root + f"{index}.png" + crop_img_paths.append(crop_img_file) + + np.savetxt(osp.join(paths['M_dir'], '{}.txt'.format(index)), trans_crop_to_full) + np.savetxt(save_intrin_path, K_crop) + cv2.imwrite(crop_img_file, image_crop) + + return crop_img_paths + +def rename_intrinsics_file(paths): with open(paths['orig_intrin_file'], 'r') as f: lines = [l.strip() for l in f.readlines() if len(l) > 0 and l[0] != '#'] eles = [[float(e) for e in l.split(',')] for l in lines] @@ -200,6 +250,13 @@ def data_process_anno(data_dir, downsample_rate=1, hw=512, AR_annotations=True): with open(paths['final_intrin_file'], 'w') as f: f.write('fx: {0}\nfy: {1}\ncx: {2}\ncy: {3}'.format(fx, fy, cx, cy)) + return fx, fy, cx, cy + +def data_process_anno(data_dir, downsample_rate=1, hw=512, AR_annotations=True): + + paths = get_arkit_default_path(data_dir, AR_annotations) + fx, fy, cx, cy = rename_intrinsics_file(paths) + K_homo = np.array([ [fx, 0, cx, 0], [0, fy, cy, 0], From 20ae9fb935fea10f4eed90ac976bdd994146d612 Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Thu, 15 Dec 2022 20:01:28 +0100 Subject: [PATCH 02/16] last --- run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/run.py b/run.py index 5fe4829..d1f550f 100644 --- a/run.py +++ b/run.py @@ -110,7 +110,6 @@ def merge_anno(cfg): def sfm(cfg): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir - #cfg.sfm.down_ratio = 1 down_ratio = cfg.sfm.down_ratio data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs From 8fe54585bcf350462d82cf8c961ad4f96a6dccdf Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Thu, 15 Dec 2022 20:11:50 +0100 Subject: [PATCH 03/16] select the mode to crop --- run.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index d1f550f..d0580f3 100644 --- a/run.py +++ b/run.py @@ -107,7 +107,7 @@ def merge_anno(cfg): json.dump(instance, f) -def sfm(cfg): +def sfm(cfg, crop_images=True): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir down_ratio = cfg.sfm.down_ratio @@ -159,9 +159,11 @@ def sfm(cfg): img_paths = [] for sub_dir in sub_dirs: img_paths += glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True) + if not os.path.exists(paths['intrin_dir']): + os.makedirs(paths['intrin_dir']) K, _ = data_utils.get_K(intrinsics_path) - for index, _ in enumerate(img_paths): - np.savetxt(paths['intrin_dir'] + f"{index}.txt", K) + for index, _ in enumerate(img_paths): + np.savetxt(paths['intrin_dir'] + f"{index}.txt", K) if len(img_paths) == 0: logger.info(f"No png image in {root_dir}") From a51fbd19126faf87468ff3b987a6b9cbe3c10cef Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Thu, 15 Dec 2022 20:18:22 +0100 Subject: [PATCH 04/16] adding option for hololens --- .hydra/config.yaml | 1 + .hydra/hydra.yaml | 3 ++- .hydra/overrides.yaml | 1 + run.py | 13 ++++++------- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.hydra/config.yaml b/.hydra/config.yaml index 5163ec6..fc86be7 100644 --- a/.hydra/config.yaml +++ b/.hydra/config.yaml @@ -19,3 +19,4 @@ sfm: covis_num: 10 rotation_thresh: 50 disable_lightning_logs: true +hololens: true diff --git a/.hydra/hydra.yaml b/.hydra/hydra.yaml index 409e264..d35400e 100644 --- a/.hydra/hydra.yaml +++ b/.hydra/hydra.yaml @@ -113,9 +113,10 @@ hydra: - dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger tiger-annotate - dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger + - +hololens=true job: name: run - override_dirname: +preprocess=sfm_spp_spg_demo,dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger + override_dirname: +hololens=true,+preprocess=sfm_spp_spg_demo,dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger tiger-annotate,dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger id: ??? num: ??? diff --git a/.hydra/overrides.yaml b/.hydra/overrides.yaml index f309db8..ec1000a 100644 --- a/.hydra/overrides.yaml +++ b/.hydra/overrides.yaml @@ -2,3 +2,4 @@ - dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger tiger-annotate - dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger +- +hololens=true diff --git a/run.py b/run.py index d0580f3..8468276 100644 --- a/run.py +++ b/run.py @@ -107,10 +107,11 @@ def merge_anno(cfg): json.dump(instance, f) -def sfm(cfg, crop_images=True): +def sfm(cfg): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir down_ratio = cfg.sfm.down_ratio + crop_images = True if cfg.hololens else False data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs for data_dir in data_dirs: @@ -129,10 +130,10 @@ def sfm(cfg, crop_images=True): paths['final_intrin_file'] = intrinsics_path paths['reproj_box_dir'] = seq_dir + "/reproj_box/" + paths['crop_img_root'] = seq_dir + "/color/" paths['intrin_dir'] = seq_dir + "/intrin/" paths['img_list'] = full_res_img_paths paths['M_dir'] = seq_dir + "/modified_poses/" - paths['crop_img_root'] = seq_dir + "/color/" obj_name = root_dir.split("/")[-1] @@ -146,13 +147,12 @@ def sfm(cfg, crop_images=True): poses_paths=poses_paths, data_root=root_dir, device="cpu", - root_2d_bbox=paths['reproj_box_dir'], step=1, hololens=cfg.hololens, + root_2d_bbox=paths['reproj_box_dir'], ) - # Crop images and save them if color folder is empty - crop_images=True + # Crop images and save them if you have MINIMAL folder structure if crop_images: img_paths = parse_images(paths, downsample_rate=1, hw=512) else: @@ -169,8 +169,7 @@ def sfm(cfg, crop_images=True): logger.info(f"No png image in {root_dir}") continue - # Choose less images from the list to build the sfm model - + # Choose less images from the list, to build the sfm model down_img_lists = [] for img_file in img_paths: index = int(img_file.split("/")[-1].split(".")[0]) From 0452cadd9fd97004131eac8b8ad15d08544da9e9 Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Thu, 15 Dec 2022 20:36:55 +0100 Subject: [PATCH 05/16] right color of images saved --- src/utils/parse_scanned_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/parse_scanned_data.py b/src/utils/parse_scanned_data.py index bc6a5ea..df18333 100644 --- a/src/utils/parse_scanned_data.py +++ b/src/utils/parse_scanned_data.py @@ -237,7 +237,7 @@ def parse_images(paths, downsample_rate=5, hw=512): np.savetxt(osp.join(paths['M_dir'], '{}.txt'.format(index)), trans_crop_to_full) np.savetxt(save_intrin_path, K_crop) - cv2.imwrite(crop_img_file, image_crop) + cv2.imwrite(crop_img_file, cv2.cvtColor(image_crop, cv2.COLOR_RGB2BGR)) return crop_img_paths From 4c02a5b417676ddb93be38a8a6dc4ddee485c0ce Mon Sep 17 00:00:00 2001 From: Ale-Burzio Date: Fri, 16 Dec 2022 10:20:23 +0100 Subject: [PATCH 06/16] remove shit --- .hydra/config.yaml | 22 ------ .hydra/hydra.yaml | 160 ------------------------------------------ .hydra/overrides.yaml | 5 -- 3 files changed, 187 deletions(-) delete mode 100644 .hydra/config.yaml delete mode 100644 .hydra/hydra.yaml delete mode 100644 .hydra/overrides.yaml diff --git a/.hydra/config.yaml b/.hydra/config.yaml deleted file mode 100644 index fc86be7..0000000 --- a/.hydra/config.yaml +++ /dev/null @@ -1,22 +0,0 @@ -work_dir: ${hydra:runtime.cwd} -data_dir: ${work_dir}/data -print_config: true -type: sfm -redo: true -dataset: - max_num_kp3d: 2500 - max_num_kp2d: 1000 - data_dir: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-annotate - outputs_dir: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger -network: - detection: superpoint - detection_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth - matching: superglue - matching_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth -sfm: - down_ratio: 5 - covis_num: 10 - rotation_thresh: 50 -disable_lightning_logs: true -hololens: true diff --git a/.hydra/hydra.yaml b/.hydra/hydra.yaml deleted file mode 100644 index d35400e..0000000 --- a/.hydra/hydra.yaml +++ /dev/null @@ -1,160 +0,0 @@ -hydra: - run: - dir: ${work_dir} - sweep: - dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} - subdir: ${hydra.job.num} - launcher: - _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher - sweeper: - _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper - max_batch_size: null - help: - app_name: ${hydra.job.name} - header: '${hydra.help.app_name} is powered by Hydra. - - ' - footer: 'Powered by Hydra (https://hydra.cc) - - Use --hydra-help to view Hydra specific help - - ' - template: '${hydra.help.header} - - == Configuration groups == - - Compose your configuration from those groups (group=option) - - - $APP_CONFIG_GROUPS - - - == Config == - - Override anything in the config (foo.bar=value) - - - $CONFIG - - - ${hydra.help.footer} - - ' - hydra_help: - template: 'Hydra (${hydra.runtime.version}) - - See https://hydra.cc for more info. - - - == Flags == - - $FLAGS_HELP - - - == Configuration groups == - - Compose your configuration from those groups (For example, append hydra/job_logging=disabled - to command line) - - - $HYDRA_CONFIG_GROUPS - - - Use ''--cfg hydra'' to Show the Hydra config. - - ' - hydra_help: ??? - hydra_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][HYDRA] %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - root: - level: INFO - handlers: - - console - loggers: - logging_example: - level: DEBUG - disable_existing_loggers: false - job_logging: - version: 1 - formatters: - simple: - format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' - handlers: - console: - class: logging.StreamHandler - formatter: simple - stream: ext://sys.stdout - file: - class: logging.FileHandler - formatter: simple - filename: ${hydra.job.name}.log - root: - level: INFO - handlers: - - console - - file - disable_existing_loggers: false - env: {} - searchpath: [] - callbacks: {} - output_subdir: .hydra - overrides: - hydra: [] - task: - - +preprocess=sfm_spp_spg_demo - - dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-annotate - - dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger - - +hololens=true - job: - name: run - override_dirname: +hololens=true,+preprocess=sfm_spp_spg_demo,dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-annotate,dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger - id: ??? - num: ??? - config_name: config.yaml - env_set: {} - env_copy: [] - config: - override_dirname: - kv_sep: '=' - item_sep: ',' - exclude_keys: [] - runtime: - version: 1.1.1 - cwd: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose - config_sources: - - path: hydra.conf - schema: pkg - provider: hydra - - path: /Users/PELLERITO/Desktop/mixed_reality_code/OnePose/configs - schema: file - provider: main - - path: '' - schema: structured - provider: schema - choices: - preprocess: sfm_spp_spg_demo - logger: null - callbacks: null - datamodule: null - model: null - trainer: null - hydra/env: default - hydra/callbacks: null - hydra/job_logging: default - hydra/hydra_logging: default - hydra/hydra_help: default - hydra/help: default - hydra/sweeper: basic - hydra/launcher: basic - hydra/output: default - verbose: false diff --git a/.hydra/overrides.yaml b/.hydra/overrides.yaml deleted file mode 100644 index ec1000a..0000000 --- a/.hydra/overrides.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- +preprocess=sfm_spp_spg_demo -- dataset.data_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/onepose_datasets/tiger - tiger-annotate -- dataset.outputs_dir=/Users/PELLERITO/Desktop/mixed_reality_code/OnePose/data/sfm_model/tiger -- +hololens=true From fe97b199bd68b534608059978afe3b57d828bf57 Mon Sep 17 00:00:00 2001 From: Ale-Burzio Date: Fri, 16 Dec 2022 10:52:08 +0100 Subject: [PATCH 07/16] fix functions --- run.py | 25 +++++++++++++------------ src/bbox_3D_estimation/utils.py | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/run.py b/run.py index 8468276..79b3282 100644 --- a/run.py +++ b/run.py @@ -2,6 +2,7 @@ import os import glob import hydra +import torch import numpy as np import os.path as osp @@ -121,19 +122,18 @@ def sfm(cfg): # Parse image, intrinsics and poses directories: poses_paths, full_res_img_paths = [], [] paths = {} - for sub_dir in sub_dirs: - seq_dir = str(Path(osp.join(root_dir, sub_dir))) + seq_dir = str(Path(osp.join(root_dir, sub_dirs[0]))) - full_res_img_paths += glob.glob(seq_dir + "/color_full/*.png", recursive=True) - poses_paths += glob.glob(seq_dir + "/poses/*.txt", recursive=True) - intrinsics_path = seq_dir + "/intrinsics.txt" + full_res_img_paths += glob.glob(seq_dir + "/color_full/*.png", recursive=True) + poses_paths += glob.glob(seq_dir + "/poses/*.txt", recursive=True) + intrinsics_path = seq_dir + "/intrinsics.txt" - paths['final_intrin_file'] = intrinsics_path - paths['reproj_box_dir'] = seq_dir + "/reproj_box/" - paths['crop_img_root'] = seq_dir + "/color/" - paths['intrin_dir'] = seq_dir + "/intrin/" - paths['img_list'] = full_res_img_paths - paths['M_dir'] = seq_dir + "/modified_poses/" + paths['final_intrin_file'] = intrinsics_path + paths['reproj_box_dir'] = seq_dir + "/reproj_box/" + paths['crop_img_root'] = seq_dir + "/color/" + paths['intrin_dir'] = seq_dir + "/intrin/" + paths['img_list'] = full_res_img_paths + paths['M_dir'] = seq_dir + "/modified_poses/" obj_name = root_dir.split("/")[-1] @@ -146,7 +146,8 @@ def sfm(cfg): full_res_img_paths=full_res_img_paths, poses_paths=poses_paths, data_root=root_dir, - device="cpu", + seq_dir=seq_dir, + device="cuda" if torch.cuda.is_available() else "cpu", step=1, hololens=cfg.hololens, root_2d_bbox=paths['reproj_box_dir'], diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 71b55ba..328712f 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -137,7 +137,7 @@ def predict_3D_bboxes( if not os.path.exists(root_2d_bbox): os.makedirs(root_2d_bbox) BboxPredictor.save_2d_bbox(file_path = root_2d_bbox + f"{id}.txt") - DetectorBox3D.add_view(bbox_orig_res, poses) + DetectorBox3D.add_view(bbox_t=bbox_orig_res, pose_t=poses, poses_orig=poses_orig) DetectorBox3D.detect_3D_box() DetectorBox3D.save_3D_box(data_root) From db877286e6d67b14abd0ce70923ec61c1b29e8de Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Fri, 16 Dec 2022 11:26:05 +0100 Subject: [PATCH 08/16] using flipped poses if hololens --- run.py | 4 +--- src/bbox_3D_estimation/utils.py | 6 ++++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/run.py b/run.py index 79b3282..a477d8a 100644 --- a/run.py +++ b/run.py @@ -157,9 +157,7 @@ def sfm(cfg): if crop_images: img_paths = parse_images(paths, downsample_rate=1, hw=512) else: - img_paths = [] - for sub_dir in sub_dirs: - img_paths += glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True) + img_paths = glob.glob(str(Path(seq_dir)) + "/color/*.png", recursive=True) if not os.path.exists(paths['intrin_dir']): os.makedirs(paths['intrin_dir']) K, _ = data_utils.get_K(intrinsics_path) diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 328712f..7a005bf 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -95,9 +95,11 @@ def shift_centres(self): shifted_poses.append(original) self.shifted_poses = shifted_poses - def save_poses(self, seq_dir): + def save_poses(self, seq_dir, hololens): """Saves poses in the OnePose format (which is inverted respect to the Hololens format)""" shift_pose_dir = f"{seq_dir}/poses_shifted/" + if hololens: + shift_pose_dir = f"{seq_dir}/poses/" # Overwrite poses os.makedirs(shift_pose_dir, exist_ok=True) for idx, pose in enumerate(self.shifted_poses): np.savetxt(f"{shift_pose_dir}{idx}.txt", pose, delimiter=" ") @@ -142,7 +144,7 @@ def predict_3D_bboxes( DetectorBox3D.detect_3D_box() DetectorBox3D.save_3D_box(data_root) DetectorBox3D.shift_centres() - DetectorBox3D.save_poses(seq_dir) + DetectorBox3D.save_poses(seq_dir, hololens) DetectorBox3D.save_dimensions(data_root) From ffb010b930dbe80b014833400bfd767c04970889 Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Fri, 16 Dec 2022 15:54:55 +0100 Subject: [PATCH 09/16] debug option --- run.py | 17 +++++++------- src/bbox_3D_estimation/utils.py | 40 ++++++++++++++------------------- src/utils/parse_scanned_data.py | 11 ++++----- 3 files changed, 32 insertions(+), 36 deletions(-) diff --git a/run.py b/run.py index a477d8a..938caba 100644 --- a/run.py +++ b/run.py @@ -177,7 +177,7 @@ def sfm(cfg): # Begin SfM and postprocess: sfm_core(cfg, down_img_lists, outputs_dir_root) - postprocess(cfg, down_img_lists, root_dir, outputs_dir_root) + postprocess(cfg, down_img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=False) def sfm_core(cfg, img_lists, outputs_dir_root): @@ -218,7 +218,7 @@ def sfm_core(cfg, img_lists, outputs_dir_root): ) # Reconstruct 3D point cloud with known image poses: - generate_empty.generate_model(img_lists, empty_dir) + generate_empty.generate_model(img_lists, empty_dir, do_ba=True) triangulation.main( deep_sfm_dir, empty_dir, @@ -227,17 +227,14 @@ def sfm_core(cfg, img_lists, outputs_dir_root): feature_out, matches_out, image_dir=None, - skip_geometric_verification=True, + skip_geometric_verification=False, ) -def postprocess(cfg, img_lists, root_dir, outputs_dir_root): +def postprocess(cfg, img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=True): """Filter points and average feature""" from src.sfm.postprocess import filter_points, feature_process, filter_tkl - # Probably here is where they use the first box - bbox_path = osp.join(root_dir, "box3d_corners.txt") - # Construct output directory structure: outputs_dir = osp.join( outputs_dir_root, @@ -255,8 +252,12 @@ def postprocess(cfg, img_lists, root_dir, outputs_dir_root): model_path, points_count_list, track_length, outputs_dir ) # For visualization only + # Probably here is where they use the first box + bbox_path = osp.join(root_dir, "box3d_corners.txt") + # Leverage the selected feature track length threshold and 3D BBox to filter 3D points: - xyzs, points_idxs = filter_points.filter_3d(model_path, track_length, bbox_path) + mask_filtering = not filter_with_3d_bbox + xyzs, points_idxs = filter_points.filter_3d(model_path, track_length, bbox_path, mask_filtering=mask_filtering) # Merge 3d points by distance between points merge_xyzs, merge_idxs = filter_points.merge(xyzs, points_idxs, dist_threshold=1e-3) diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 7a005bf..9b3c27c 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -34,7 +34,7 @@ def add_view(self, bbox_t: np.ndarray, pose_t: np.ndarray, poses_orig: list): self.poses_list.append(poses_orig) - def detect_3D_box(self): + def detect_3D_box(self, plot_3dbbox=False): object_idx = 0 selected_frames = self.bboxes.shape[0] self.visibility = np.ones((selected_frames, 1)) @@ -48,12 +48,6 @@ def detect_3D_box(self): # Coordinates of the points mins and maxs points = np.array(list(itertools.product(*zip(mins, maxs)))) - # Points in the camera frame - # points = np.dot(points, R.T) - - # Shift correctly the parralelepiped (we want it centered in the origin) - # points[:, 0:3] = np.add(centre[None, :], points[:, :3]) - self.axes = axes self.points = points self.centre = centre @@ -61,26 +55,25 @@ def detect_3D_box(self): # Transformation to have coordinates centered in the bounding box (and aligned with it) M = np.empty((4, 4)) - M[:3, :3] = R + #M[:3, :3] = R + M[:3, :3] = np.eye(3) M[:3, 3] = centre M[3, :] = [0, 0, 0, 1] - # print(M) self.M = np.linalg.inv(M) - - # gt_p = np.loadtxt(f"data/onepose_datasets/val_data/0606-tiger-others/box3d_corners_GT.txt") - - # plot_3D_scene( - # estQs=estQs, - # gtQs=gt_p, - # Ms_t=self.poses, - # dataset="tiger", - # save_output_images=False, - # points=points, - # GT_points=gt_p - # ) - # plt.show() + if plot_3dbbox: + gt_p = np.loadtxt(f"data/onepose_datasets/val_data/0606-tiger-others/box3d_corners_GT.txt") + plot_3D_scene( + estQs=estQs, + gtQs=gt_p, + Ms_t=self.poses, + dataset="tiger", + save_output_images=False, + points=points, + GT_points=gt_p + ) + plt.show() def save_3D_box(self, data_root): @@ -158,7 +151,8 @@ def read_list_poses(list, hololens=False): for idx, file_path in enumerate(list): with open(file_path) as f_input: if hololens: - pose = np.transpose(np.linalg.inv(np.loadtxt(f_input))[:3, :]) # TODO poses are inverted when from hololens + # TODO poses are inverted when from hololens + pose = np.transpose(np.linalg.inv(np.loadtxt(f_input))[:3, :]) else: pose = np.transpose(np.loadtxt(f_input)[:3, :]) if idx == 0: diff --git a/src/utils/parse_scanned_data.py b/src/utils/parse_scanned_data.py index df18333..d469ca9 100644 --- a/src/utils/parse_scanned_data.py +++ b/src/utils/parse_scanned_data.py @@ -190,7 +190,7 @@ def parse_video(paths, downsample_rate=5, bbox_3d_homo=None, hw=512, AR_annotati index += 1 cap.release() -def parse_images(paths, downsample_rate=5, hw=512): +def parse_images(paths, downsample_rate=5, hw=512, save_rotations=False): orig_intrin_file = paths['final_intrin_file'] crop_img_root = paths['crop_img_root'] intrin_dir = paths['intrin_dir'] @@ -229,13 +229,14 @@ def parse_images(paths, downsample_rate=5, hw=512): K_crop, K_crop_homo = data_utils.get_K_crop_resize(box_new, K_crop, resize_shape) image_crop, trans2 = data_utils.get_image_crop_resize(image_crop, box_new, resize_shape) - trans_full_to_crop = trans2 @ trans1 - trans_crop_to_full = np.linalg.inv(trans_full_to_crop) - crop_img_file = crop_img_root + f"{index}.png" crop_img_paths.append(crop_img_file) - np.savetxt(osp.join(paths['M_dir'], '{}.txt'.format(index)), trans_crop_to_full) + if save_rotations: + trans_full_to_crop = trans2 @ trans1 + trans_crop_to_full = np.linalg.inv(trans_full_to_crop) + np.savetxt(osp.join(paths['M_dir'], '{}.txt'.format(index)), trans_crop_to_full) + np.savetxt(save_intrin_path, K_crop) cv2.imwrite(crop_img_file, cv2.cvtColor(image_crop, cv2.COLOR_RGB2BGR)) From eae12a3c5898f183c8550469db6689bfd43f31de Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Sun, 18 Dec 2022 15:01:16 +0100 Subject: [PATCH 10/16] first committ --- configs/experiment/test_demo.yaml | 9 ++--- configs/preprocess/sfm_spp_spg_val.yaml | 12 +++---- run.py | 13 ++++--- shift_poses.py | 46 +++++++++++++++++++++---- src/bbox_3D_estimation/plotting.py | 8 ++--- src/bbox_3D_estimation/utils.py | 45 ++++++++++++++---------- 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/configs/experiment/test_demo.yaml b/configs/experiment/test_demo.yaml index 5516888..4b37a64 100644 --- a/configs/experiment/test_demo.yaml +++ b/configs/experiment/test_demo.yaml @@ -13,7 +13,7 @@ model: extractor_model_path: ${work_dir}/data/models/extractors/SuperPoint/superpoint_v1.pth match_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth -scan_data_dir: ${data_dir}/onepose_datasets/val_data +scan_data_dir: ${work_dir}/data/costum_datasets/test sfm_model_dir: ${data_dir}/sfm_model network: @@ -24,10 +24,11 @@ max_num_kp3d: 2500 input: data_dirs: - - ${scan_data_dir}/0606-tiger-others tiger-1 - - ${scan_data_dir}/0609-doll-others doll-3 + - ${scan_data_dir}/demo_bottle_sfm bottle-1 + # - ${scan_data_dir}/0606-tiger-others tiger-1 + # - ${scan_data_dir}/0609-doll-others doll-3 sfm_model_dirs: - - ${sfm_model_dir}/0606-tiger-others + - ${sfm_model_dir}/demo_bottle_sfm output: vis_dir: ${work_dir}/runs/vis/demo eval_dir: ${work_dir}/runs/eval/demo \ No newline at end of file diff --git a/configs/preprocess/sfm_spp_spg_val.yaml b/configs/preprocess/sfm_spp_spg_val.yaml index ef0f599..375ef36 100644 --- a/configs/preprocess/sfm_spp_spg_val.yaml +++ b/configs/preprocess/sfm_spp_spg_val.yaml @@ -3,18 +3,18 @@ type: sfm work_dir: ${hydra:runtime.cwd} redo: True -hololens: True +hololens: False -# scan_data_dir: ${work_dir}/data/onepose_datasets/val_data -scan_data_dir: ${work_dir}/data/costum_datasets/test +scan_data_dir: ${work_dir}/data/onepose_datasets/val_data +# scan_data_dir: ${work_dir}/data/costum_datasets/test dataset: max_num_kp3d: 2500 max_num_kp2d: 1000 data_dir: - # - ${scan_data_dir}/0606-tiger-others tiger-2 - - ${scan_data_dir}/demo_capture demo-1 + - ${scan_data_dir}/0606-tiger-others tiger-2 + # - ${scan_data_dir}/demo_bottle_sfm bottle-1 outputs_dir: ${work_dir}/data/sfm_model/{} @@ -27,7 +27,7 @@ network: matching_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth sfm: - down_ratio: 5 + down_ratio: 1 covis_num: 10 rotation_thresh: 50 diff --git a/run.py b/run.py index 938caba..b2d56c2 100644 --- a/run.py +++ b/run.py @@ -4,6 +4,7 @@ import hydra import torch import numpy as np +import os import os.path as osp from loguru import logger @@ -112,7 +113,7 @@ def sfm(cfg): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir down_ratio = cfg.sfm.down_ratio - crop_images = True if cfg.hololens else False + crop_images = False data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs for data_dir in data_dirs: @@ -128,6 +129,9 @@ def sfm(cfg): poses_paths += glob.glob(seq_dir + "/poses/*.txt", recursive=True) intrinsics_path = seq_dir + "/intrinsics.txt" + poses_paths.sort(key=lambda i: int(os.path.splitext(os.path.basename(i))[0])) + full_res_img_paths.sort(key=lambda i: int(os.path.splitext(os.path.basename(i))[0])) + paths['final_intrin_file'] = intrinsics_path paths['reproj_box_dir'] = seq_dir + "/reproj_box/" paths['crop_img_root'] = seq_dir + "/color/" @@ -167,17 +171,18 @@ def sfm(cfg): if len(img_paths) == 0: logger.info(f"No png image in {root_dir}") continue - + + img_paths.sort(key=lambda i: int(os.path.splitext(os.path.basename(i))[0])) # Choose less images from the list, to build the sfm model down_img_lists = [] for img_file in img_paths: index = int(img_file.split("/")[-1].split(".")[0]) if index % down_ratio == 0: down_img_lists.append(img_file) - + down_img_lists = sorted(down_img_lists, key=lambda i: int(os.path.splitext(os.path.basename(i))[0])) # Begin SfM and postprocess: sfm_core(cfg, down_img_lists, outputs_dir_root) - postprocess(cfg, down_img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=False) + postprocess(cfg, down_img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=True) def sfm_core(cfg, img_lists, outputs_dir_root): diff --git a/shift_poses.py b/shift_poses.py index c0978f2..61c7d87 100644 --- a/shift_poses.py +++ b/shift_poses.py @@ -3,27 +3,59 @@ import numpy as np from src.bbox_3D_estimation.utils import read_list_poses_orig import re +from scipy.spatial.transform import Rotation as R + + +# r = R.from_quat([0, 0, np.sin(np.pi/4), np.cos(np.pi/4)]) +# print(r.as_matrix()) regex = re.compile('[^0-9]') -DIR = "data/onepose_datasets/val_data/0606-tiger-others/tiger-2/" +DIR = "data/costum_datasets/test/demo_bottle_sfm/bottle-1/" -poses = sorted(glob.glob(f"{DIR}backup/poses/*.txt")) +poses = sorted(glob.glob(f"{DIR}backup-poses/poses/*.txt")) names = [] for pose in poses: - name = regex.sub("", pose)[5:] + name = regex.sub("", pose)[1:] names.append(name) poses = read_list_poses_orig(poses) - +M = np.empty((4, 4)) +M[:3, :3] = np.array([[ -1.0000000, 0.0000000, 0.0000000], + [ 0.0000000, -1.0000000, -0.0000000], + [ 0.0000000, 0.0000000, 1.0000000 ]]) +M[:3, 3] = np.array([+0, 0, 0]) +M[3, :] = [0, 0, 0, 1] shifted_poses = [] for pose in poses: + + pose = np.dot(M, pose) inverted = np.linalg.inv(pose) - inverted[0:3,3] += np.array([+5, -10, +10]) - original = np.linalg.inv(inverted) - shifted_poses.append(original) + # Extract rotation matrix and translation vector from left-handed pose T + # R = pose[:3, :3] + # t = pose[:3, 3] + + # # Negate last element of translation vector + # t[2] = -t[2] + + # # Convert the pose to a right-handed coordinate system + # R_right = np.array([[R[0,0], R[0,1], -R[0,2]], + # [R[1,0], R[1,1], -R[1,2]], + # [-R[2,0], -R[2,1], R[2,2]]]) + + # # Construct right-handed pose T' + # T_prime = np.eye(4) + # T_prime[:3, :3] = R_right + # T_prime[:3, 3] = t + + + # inverted = np.dot(M, inverted) + # T_prime = pose + # original = np.linalg.inv(inverted) + + shifted_poses.append(inverted) shift_pose_dir = f"{DIR}poses/" diff --git a/src/bbox_3D_estimation/plotting.py b/src/bbox_3D_estimation/plotting.py index 2b2ff21..9dfdd20 100644 --- a/src/bbox_3D_estimation/plotting.py +++ b/src/bbox_3D_estimation/plotting.py @@ -180,10 +180,10 @@ def plot_camera(M, figure_axes): y = points[1, :].reshape(2, 5) z = points[2, :].reshape(2, 5) - # figure_axes.plot_wireframe( - # x, y, z, rstride=1, cstride=1, color=[0, 0, 0], linewidth=0.5 - # ) - figure_axes.scatter(t[0], t[1], t[2]) + figure_axes.plot_wireframe( + x, y, z, rstride=1, cstride=1, color=[0, 0, 0], linewidth=0.5 + ) + # figure_axes.scatter(t[0], t[1], t[2]) def plot_3D_scene( diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 9b3c27c..c6e0311 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -52,28 +52,15 @@ def detect_3D_box(self, plot_3dbbox=False): self.points = points self.centre = centre self.R = R + self.estQs = estQs # Transformation to have coordinates centered in the bounding box (and aligned with it) M = np.empty((4, 4)) - #M[:3, :3] = R - M[:3, :3] = np.eye(3) + M[:3, :3] = R M[:3, 3] = centre M[3, :] = [0, 0, 0, 1] - self.M = np.linalg.inv(M) - - if plot_3dbbox: - gt_p = np.loadtxt(f"data/onepose_datasets/val_data/0606-tiger-others/box3d_corners_GT.txt") - plot_3D_scene( - estQs=estQs, - gtQs=gt_p, - Ms_t=self.poses, - dataset="tiger", - save_output_images=False, - points=points, - GT_points=gt_p - ) - plt.show() + self.M = np.linalg.inv(M) def save_3D_box(self, data_root): @@ -90,7 +77,7 @@ def shift_centres(self): def save_poses(self, seq_dir, hololens): """Saves poses in the OnePose format (which is inverted respect to the Hololens format)""" - shift_pose_dir = f"{seq_dir}/poses_shifted/" + shift_pose_dir = f"{seq_dir}/poses/" if hololens: shift_pose_dir = f"{seq_dir}/poses/" # Overwrite poses os.makedirs(shift_pose_dir, exist_ok=True) @@ -100,6 +87,19 @@ def save_poses(self, seq_dir, hololens): def save_dimensions(self, data_root): np.savetxt(data_root + "/box3d_dimensions.txt", self.axes, delimiter=" ") + def plot_3D_bb(self, poses): + plot_3D_scene( + estQs=self.estQs, + gtQs=None, + Ms_t=poses, + dataset="tiger", + save_output_images=False, + points=self.points, + GT_points=None + ) + plt.show() + + def predict_3D_bboxes( full_res_img_paths, @@ -139,6 +139,15 @@ def predict_3D_bboxes( DetectorBox3D.shift_centres() DetectorBox3D.save_poses(seq_dir, hololens) DetectorBox3D.save_dimensions(data_root) + poses_t=None + for id, img_path in enumerate(tqdm(full_res_img_paths)): + poses = read_list_poses([poses_paths[id]], hololens=False) + if poses_t is None: + poses_t = poses + else: + poses_t = np.vstack((poses_t, poses)) + + DetectorBox3D.plot_3D_bb(poses_t) def sort_path_list(path_list): @@ -166,7 +175,7 @@ def read_list_poses_orig(list): poses = [] for idx, file_path in enumerate(list): with open(file_path) as f_input: - poses.append(np.linalg.inv(np.loadtxt(f_input))) + poses.append(np.loadtxt(f_input)) return poses From a700715385ff6b3ecc419d72040b9faacdfe2662 Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Sun, 18 Dec 2022 20:57:47 +0100 Subject: [PATCH 11/16] from left to right coordinate system --- configs/preprocess/sfm_spp_spg_val.yaml | 10 +++--- shift_poses.py | 45 ++++++++++++++++++------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/configs/preprocess/sfm_spp_spg_val.yaml b/configs/preprocess/sfm_spp_spg_val.yaml index 375ef36..22af463 100644 --- a/configs/preprocess/sfm_spp_spg_val.yaml +++ b/configs/preprocess/sfm_spp_spg_val.yaml @@ -5,16 +5,16 @@ work_dir: ${hydra:runtime.cwd} redo: True hololens: False -scan_data_dir: ${work_dir}/data/onepose_datasets/val_data -# scan_data_dir: ${work_dir}/data/costum_datasets/test +# scan_data_dir: ${work_dir}/data/onepose_datasets/val_data +scan_data_dir: ${work_dir}/data/costum_datasets/test dataset: max_num_kp3d: 2500 max_num_kp2d: 1000 data_dir: - - ${scan_data_dir}/0606-tiger-others tiger-2 - # - ${scan_data_dir}/demo_bottle_sfm bottle-1 + # - ${scan_data_dir}/0606-tiger-others tiger-2 + - ${scan_data_dir}/demo_bottle_sfm bottle-1 outputs_dir: ${work_dir}/data/sfm_model/{} @@ -27,7 +27,7 @@ network: matching_model_path: ${work_dir}/data/models/matchers/SuperGlue/superglue_outdoor.pth sfm: - down_ratio: 1 + down_ratio: 3 covis_num: 10 rotation_thresh: 50 diff --git a/shift_poses.py b/shift_poses.py index 61c7d87..6de5a66 100644 --- a/shift_poses.py +++ b/shift_poses.py @@ -3,7 +3,8 @@ import numpy as np from src.bbox_3D_estimation.utils import read_list_poses_orig import re -from scipy.spatial.transform import Rotation as R +from scipy.spatial.transform import Rotation as Rotmat + # r = R.from_quat([0, 0, np.sin(np.pi/4), np.cos(np.pi/4)]) @@ -21,29 +22,45 @@ poses = read_list_poses_orig(poses) M = np.empty((4, 4)) -M[:3, :3] = np.array([[ -1.0000000, 0.0000000, 0.0000000], - [ 0.0000000, -1.0000000, -0.0000000], - [ 0.0000000, 0.0000000, 1.0000000 ]]) +M[:3, :3] = np.array([[ 1.0000000, 0.0000000, 0.0000000], + [ 0.0000000, 1.0000000, 0.0000000], + [ 0.0000000, 0.0000000, 1.0000000 ]]) M[:3, 3] = np.array([+0, 0, 0]) M[3, :] = [0, 0, 0, 1] + +def ruf_to_flu(pose): + rotquat_ruf = Rotmat.from_matrix(pose[:3,:3]).as_quat() + rotquat_flu = np.array([-rotquat_ruf[2], rotquat_ruf[0], -rotquat_ruf[1], rotquat_ruf[3]]) + rotmat_flu = Rotmat.from_quat(rotquat_flu).as_matrix() + + transl_ruf = pose[:3, 3] + transl_flu = np.array([transl_ruf[2], -transl_ruf[0], transl_ruf[1]]) + + new_pose = np.eye(4) + new_pose[:3,:3] = rotmat_flu @ np.array([0,0,1,0,1,0,-1,0,0]).reshape((3,3)) + new_pose[:3, 3] = transl_flu + + return new_pose + shifted_poses = [] for pose in poses: - pose = np.dot(M, pose) - inverted = np.linalg.inv(pose) + # pose = np.dot(M, pose) + # pose = np.linalg.inv(pose) # Extract rotation matrix and translation vector from left-handed pose T # R = pose[:3, :3] # t = pose[:3, 3] # # Negate last element of translation vector - # t[2] = -t[2] + # t[0] = t[2] + # t[2] = t[0] # # Convert the pose to a right-handed coordinate system - # R_right = np.array([[R[0,0], R[0,1], -R[0,2]], - # [R[1,0], R[1,1], -R[1,2]], - # [-R[2,0], -R[2,1], R[2,2]]]) + # R_right = np.array([[R[0,2], R[0,1], R[0,0]], + # [R[1,2], R[1,1], R[1,0]], + # [R[2,2], R[2,1], R[2,0]]]) # # Construct right-handed pose T' # T_prime = np.eye(4) @@ -51,11 +68,13 @@ # T_prime[:3, 3] = t - # inverted = np.dot(M, inverted) + # inverted = T_prime #np.dot(T_prime, pose) # T_prime = pose - # original = np.linalg.inv(inverted) + pose = ruf_to_flu(pose) + + original = np.linalg.inv(pose) - shifted_poses.append(inverted) + shifted_poses.append(original) shift_pose_dir = f"{DIR}poses/" From e6ec5d145785dc0eb24622b3a83dad47d0d2c966 Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Wed, 4 Jan 2023 11:42:56 +0100 Subject: [PATCH 12/16] working --- configs/preprocess/sfm_spp_spg_val.yaml | 1 + run.py | 2 +- shift_poses.py | 28 +++++++++++++------------ src/bbox_3D_estimation/plotting.py | 3 ++- src/bbox_3D_estimation/utils.py | 12 ++++++++--- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/configs/preprocess/sfm_spp_spg_val.yaml b/configs/preprocess/sfm_spp_spg_val.yaml index 22af463..f0ab440 100644 --- a/configs/preprocess/sfm_spp_spg_val.yaml +++ b/configs/preprocess/sfm_spp_spg_val.yaml @@ -15,6 +15,7 @@ dataset: data_dir: # - ${scan_data_dir}/0606-tiger-others tiger-2 - ${scan_data_dir}/demo_bottle_sfm bottle-1 + # - ${scan_data_dir}/demo_capture demo-1 outputs_dir: ${work_dir}/data/sfm_model/{} diff --git a/run.py b/run.py index b2d56c2..e6a70fa 100644 --- a/run.py +++ b/run.py @@ -236,7 +236,7 @@ def sfm_core(cfg, img_lists, outputs_dir_root): ) -def postprocess(cfg, img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=True): +def postprocess(cfg, img_lists, root_dir, outputs_dir_root, filter_with_3d_bbox=False): """Filter points and average feature""" from src.sfm.postprocess import filter_points, feature_process, filter_tkl diff --git a/shift_poses.py b/shift_poses.py index 6de5a66..e5edaa2 100644 --- a/shift_poses.py +++ b/shift_poses.py @@ -3,18 +3,18 @@ import numpy as np from src.bbox_3D_estimation.utils import read_list_poses_orig import re -from scipy.spatial.transform import Rotation as Rotmat +from scipy.spatial.transform import Rotation as R -# r = R.from_quat([0, 0, np.sin(np.pi/4), np.cos(np.pi/4)]) -# print(r.as_matrix()) +r = R.from_quat([0.3535534, 0.3535534, 0.1464466, 0.8535534]) +print(r.as_matrix()) regex = re.compile('[^0-9]') -DIR = "data/costum_datasets/test/demo_bottle_sfm/bottle-1/" +DIR = "data/costum_datasets/test/demo_bottle/bottle-1/" -poses = sorted(glob.glob(f"{DIR}backup-poses/poses/*.txt")) +poses = sorted(glob.glob(f"{DIR}backup/poses/*.txt")) names = [] for pose in poses: name = regex.sub("", pose)[1:] @@ -22,17 +22,18 @@ poses = read_list_poses_orig(poses) M = np.empty((4, 4)) -M[:3, :3] = np.array([[ 1.0000000, 0.0000000, 0.0000000], - [ 0.0000000, 1.0000000, 0.0000000], - [ 0.0000000, 0.0000000, 1.0000000 ]]) -M[:3, 3] = np.array([+0, 0, 0]) +# M[:3, :3] = np.array([[ 1.0000000, 0.0000000, 0.0000000], +# [ 0.0000000, 1.0000000, 0.0000000], +# [ 0.0000000, 0.0000000, 1.0000000 ]]) +M[:3, :3] = r.as_matrix() +M[:3, 3] = np.array([1, 1, 0]) M[3, :] = [0, 0, 0, 1] def ruf_to_flu(pose): - rotquat_ruf = Rotmat.from_matrix(pose[:3,:3]).as_quat() + rotquat_ruf = R.from_matrix(pose[:3,:3]).as_quat() rotquat_flu = np.array([-rotquat_ruf[2], rotquat_ruf[0], -rotquat_ruf[1], rotquat_ruf[3]]) - rotmat_flu = Rotmat.from_quat(rotquat_flu).as_matrix() + rotmat_flu = R.from_quat(rotquat_flu).as_matrix() transl_ruf = pose[:3, 3] transl_flu = np.array([transl_ruf[2], -transl_ruf[0], transl_ruf[1]]) @@ -46,8 +47,9 @@ def ruf_to_flu(pose): shifted_poses = [] for pose in poses: - # pose = np.dot(M, pose) + # pose = np.linalg.inv(pose) + # pose = np.dot(M, pose) # Extract rotation matrix and translation vector from left-handed pose T # R = pose[:3, :3] @@ -70,7 +72,7 @@ def ruf_to_flu(pose): # inverted = T_prime #np.dot(T_prime, pose) # T_prime = pose - pose = ruf_to_flu(pose) + # pose = ruf_to_flu(pose) original = np.linalg.inv(pose) diff --git a/src/bbox_3D_estimation/plotting.py b/src/bbox_3D_estimation/plotting.py index 9dfdd20..13970f8 100644 --- a/src/bbox_3D_estimation/plotting.py +++ b/src/bbox_3D_estimation/plotting.py @@ -152,7 +152,7 @@ def plot_camera(M, figure_axes): The base of the pyramid points in the positive Z axis direction. """ # Compute the points for the camera at the origin, aligned with the axes. - base_width = 0.20 # Width (and height) of the pyramid base in meters + base_width = 0.05 # Width (and height) of the pyramid base in meters x = np.array([0, 0, 0, 0, 0, 1, 1, -1, -1, 1]) * base_width / 2 y = np.array([0, 0, 0, 0, 0, 1, -1, -1, 1, 1]) * base_width / 2 z = np.array([0, 0, 0, 0, 0, 2, 2, 2, 2, 2]) * base_width / 2 @@ -184,6 +184,7 @@ def plot_camera(M, figure_axes): x, y, z, rstride=1, cstride=1, color=[0, 0, 0], linewidth=0.5 ) # figure_axes.scatter(t[0], t[1], t[2]) + figure_axes.scatter(0, 0, 0) def plot_3D_scene( diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index c6e0311..7d683c7 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -48,6 +48,12 @@ def detect_3D_box(self, plot_3dbbox=False): # Coordinates of the points mins and maxs points = np.array(list(itertools.product(*zip(mins, maxs)))) + # # Points in the camera frame + # points = np.dot(points, R.T) + + # # Shift correctly the parralelepiped + # points[:, 0:3] = np.add(centre[None, :], points[:, :3],) + self.axes = axes self.points = points self.centre = centre @@ -56,11 +62,11 @@ def detect_3D_box(self, plot_3dbbox=False): # Transformation to have coordinates centered in the bounding box (and aligned with it) M = np.empty((4, 4)) - M[:3, :3] = R - M[:3, 3] = centre + M[:3, :3] = R # np.eye(3) # R + M[:3, 3] = centre # [0, 0, 0] # centre M[3, :] = [0, 0, 0, 1] - self.M = np.linalg.inv(M) + self.M = np.linalg.inv(M) def save_3D_box(self, data_root): From 5fe5ce5059d5d49b19d47493abb1da3fc15d5b26 Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Thu, 5 Jan 2023 12:35:32 +0100 Subject: [PATCH 13/16] plot GT --- configs/experiment/object_detector_2.yaml | 18 ++++++++++-------- configs/preprocess/sfm_spp_spg_val.yaml | 7 ++++--- src/bbox_3D_estimation/utils.py | 8 +++++--- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/configs/experiment/object_detector_2.yaml b/configs/experiment/object_detector_2.yaml index 1637ba2..acce094 100644 --- a/configs/experiment/object_detector_2.yaml +++ b/configs/experiment/object_detector_2.yaml @@ -18,14 +18,16 @@ sfm_model_dir: ${data_dir}/sfm_model input: data_dirs: - - ${scan_data_dir}/0606-tiger-others tiger-2 - - ${scan_data_dir}/0613-adidasshoeright-others adidasshoeright-2 - - ${scan_data_dir}/0601-loquat-box loquat-2 - - ${scan_data_dir}/0602-aficion-box aficion-2 + - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1 + # - ${scan_data_dir}/0606-tiger-others tiger-2 + # - ${scan_data_dir}/0613-adidasshoeright-others adidasshoeright-2 + # - ${scan_data_dir}/0601-loquat-box loquat-2 + # - ${scan_data_dir}/0602-aficion-box aficion-2 sfm_model_dirs: - - ${sfm_model_dir}/0606-tiger-others - - ${sfm_model_dir}/0613-adidasshoeright-others - - ${sfm_model_dir}/0601-loquat-box - - ${sfm_model_dir}/0602-aficion-box \ No newline at end of file + - ${scan_data_dir}/0620-dinosaurcup-bottle + # - ${sfm_model_dir}/0606-tiger-others + # - ${sfm_model_dir}/0613-adidasshoeright-others + # - ${sfm_model_dir}/0601-loquat-box + # - ${sfm_model_dir}/0602-aficion-box \ No newline at end of file diff --git a/configs/preprocess/sfm_spp_spg_val.yaml b/configs/preprocess/sfm_spp_spg_val.yaml index f0ab440..556f2db 100644 --- a/configs/preprocess/sfm_spp_spg_val.yaml +++ b/configs/preprocess/sfm_spp_spg_val.yaml @@ -5,8 +5,8 @@ work_dir: ${hydra:runtime.cwd} redo: True hololens: False -# scan_data_dir: ${work_dir}/data/onepose_datasets/val_data -scan_data_dir: ${work_dir}/data/costum_datasets/test +scan_data_dir: ${work_dir}/data/onepose_datasets/val_data +# scan_data_dir: ${work_dir}/data/costum_datasets/test dataset: max_num_kp3d: 2500 @@ -14,7 +14,8 @@ dataset: data_dir: # - ${scan_data_dir}/0606-tiger-others tiger-2 - - ${scan_data_dir}/demo_bottle_sfm bottle-1 + - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1 + # - ${scan_data_dir}/demo_bottle_sfm bottle-1 # - ${scan_data_dir}/demo_capture demo-1 outputs_dir: ${work_dir}/data/sfm_model/{} diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 7d683c7..42c1f0f 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -93,7 +93,7 @@ def save_poses(self, seq_dir, hololens): def save_dimensions(self, data_root): np.savetxt(data_root + "/box3d_dimensions.txt", self.axes, delimiter=" ") - def plot_3D_bb(self, poses): + def plot_3D_bb(self, poses, GT_points=None): plot_3D_scene( estQs=self.estQs, gtQs=None, @@ -101,7 +101,7 @@ def plot_3D_bb(self, poses): dataset="tiger", save_output_images=False, points=self.points, - GT_points=None + GT_points=GT_points ) plt.show() @@ -152,8 +152,10 @@ def predict_3D_bboxes( poses_t = poses else: poses_t = np.vstack((poses_t, poses)) + + GT_points = np.loadtxt(data_root + "/box3d_corners_GT.txt") - DetectorBox3D.plot_3D_bb(poses_t) + DetectorBox3D.plot_3D_bb(poses_t, GT_points=GT_points) def sort_path_list(path_list): From 2ea40fdfd190906c5ebc9ee03af9dcf42953ce7e Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Thu, 5 Jan 2023 19:30:35 +0100 Subject: [PATCH 14/16] small changes --- shift_poses.py | 14 +++++------ src/bbox_3D_estimation/plotting.py | 24 ++++++++++++++++-- src/bbox_3D_estimation/utils.py | 39 +++++++++++++++++++++++++++--- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/shift_poses.py b/shift_poses.py index e5edaa2..f7e8a4c 100644 --- a/shift_poses.py +++ b/shift_poses.py @@ -7,17 +7,17 @@ -r = R.from_quat([0.3535534, 0.3535534, 0.1464466, 0.8535534]) -print(r.as_matrix()) +r = R.from_quat([ 0, -0.4871745, 0, -0.8733046 ]) +# print(r.as_matrix()) regex = re.compile('[^0-9]') -DIR = "data/costum_datasets/test/demo_bottle/bottle-1/" +DIR = "data/onepose_datasets/val_data/0620-dinosaurcup-bottle/dinosaurcup-1/" poses = sorted(glob.glob(f"{DIR}backup/poses/*.txt")) names = [] for pose in poses: - name = regex.sub("", pose)[1:] + name = regex.sub("", pose)[5:] names.append(name) poses = read_list_poses_orig(poses) @@ -25,8 +25,8 @@ # M[:3, :3] = np.array([[ 1.0000000, 0.0000000, 0.0000000], # [ 0.0000000, 1.0000000, 0.0000000], # [ 0.0000000, 0.0000000, 1.0000000 ]]) -M[:3, :3] = r.as_matrix() -M[:3, 3] = np.array([1, 1, 0]) +M[:3, :3] = np.eye(3) # r.as_matrix() +M[:3, 3] = np.array([0, 0, 0]) M[3, :] = [0, 0, 0, 1] @@ -48,7 +48,7 @@ def ruf_to_flu(pose): for pose in poses: - # pose = np.linalg.inv(pose) + pose = np.linalg.inv(pose) # pose = np.dot(M, pose) # Extract rotation matrix and translation vector from left-handed pose T diff --git a/src/bbox_3D_estimation/plotting.py b/src/bbox_3D_estimation/plotting.py index 13970f8..26bedf3 100644 --- a/src/bbox_3D_estimation/plotting.py +++ b/src/bbox_3D_estimation/plotting.py @@ -184,11 +184,11 @@ def plot_camera(M, figure_axes): x, y, z, rstride=1, cstride=1, color=[0, 0, 0], linewidth=0.5 ) # figure_axes.scatter(t[0], t[1], t[2]) - figure_axes.scatter(0, 0, 0) + # figure_axes.scatter(0, 0, 0) def plot_3D_scene( - estQs, gtQs, Ms_t, dataset, save_output_images, points, GT_points, visibility=None + estQs, gtQs, Ms_t, dataset, save_output_images, points, orig_points, GT_points, visibility=None ): """Plot""" fig = plt.figure(figsize=(8, 8)) # Open a new figure. @@ -260,6 +260,26 @@ def plot_3D_scene( ) figure_axes.scatter(GT_points[:, 0], GT_points[:, 1], GT_points[:, 2]) + + Z = orig_points + # list of sides' polygons of figure + verts = [ + [Z[0], Z[1], Z[3], Z[2]], + [Z[4], Z[5], Z[7], Z[6]], + [Z[2], Z[3], Z[7], Z[6]], + [Z[0], Z[1], Z[5], Z[4]], + [Z[0], Z[2], Z[6], Z[4]], + [Z[1], Z[3], Z[7], Z[5]], + ] + + # plot sides + figure_axes.add_collection3d( + Poly3DCollection( + verts, facecolors="blue", linewidths=1, edgecolors="r", alpha=0.25 + ) + ) + + figure_axes.scatter(orig_points[:, 0], orig_points[:, 1], orig_points[:, 2]) fig.show() diff --git a/src/bbox_3D_estimation/utils.py b/src/bbox_3D_estimation/utils.py index 42c1f0f..be157e0 100644 --- a/src/bbox_3D_estimation/utils.py +++ b/src/bbox_3D_estimation/utils.py @@ -48,14 +48,15 @@ def detect_3D_box(self, plot_3dbbox=False): # Coordinates of the points mins and maxs points = np.array(list(itertools.product(*zip(mins, maxs)))) - # # Points in the camera frame - # points = np.dot(points, R.T) + # Points in the camera frame + orig_points = np.dot(points, R.T) - # # Shift correctly the parralelepiped - # points[:, 0:3] = np.add(centre[None, :], points[:, :3],) + # Shift correctly the parralelepiped + orig_points[:, 0:3] = np.add(centre[None, :], orig_points[:, :3],) self.axes = axes self.points = points + self.orig_points = orig_points self.centre = centre self.R = R self.estQs = estQs @@ -101,10 +102,37 @@ def plot_3D_bb(self, poses, GT_points=None): dataset="tiger", save_output_images=False, points=self.points, + orig_points=self.orig_points, GT_points=GT_points ) plt.show() + def iou(self, coords2): + + # Predicted parallelepiped + coords1 = self.points + + # Find the min and max x, y, and z coordinates for each parallelepiped + min_coords1 = np.min(coords1, axis=0) + max_coords1 = np.max(coords1, axis=0) + min_coords2 = np.min(coords2, axis=0) + max_coords2 = np.max(coords2, axis=0) + + # Find the intersection of the two parallelepipeds + intersection_min = np.max([min_coords1, min_coords2], axis=0) + intersection_max = np.min([max_coords1, max_coords2], axis=0) + + # Find the volume of the intersection + intersection_volume = np.prod(intersection_max - intersection_min) + + # Find the volume of each parallelepiped + volume1 = np.prod(max_coords1 - min_coords1) + volume2 = np.prod(max_coords2 - min_coords2) + + # Calculate the intersection over union (IoU) + iou = intersection_volume / (volume1 + volume2 - intersection_volume) + + return iou def predict_3D_bboxes( @@ -155,6 +183,9 @@ def predict_3D_bboxes( GT_points = np.loadtxt(data_root + "/box3d_corners_GT.txt") + iou = DetectorBox3D.iou(GT_points) + print("IoU is: ", iou) + print("Predicted transformation is: \n", np.linalg.inv(DetectorBox3D.M)) DetectorBox3D.plot_3D_bb(poses_t, GT_points=GT_points) From feba93824499f6f2077ea92928f4f05b3fa39894 Mon Sep 17 00:00:00 2001 From: Pippo809 Date: Thu, 5 Jan 2023 23:36:41 +0100 Subject: [PATCH 15/16] setup --- configs/preprocess/sfm_spp_spg_val.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/preprocess/sfm_spp_spg_val.yaml b/configs/preprocess/sfm_spp_spg_val.yaml index 556f2db..eb7b93a 100644 --- a/configs/preprocess/sfm_spp_spg_val.yaml +++ b/configs/preprocess/sfm_spp_spg_val.yaml @@ -13,8 +13,8 @@ dataset: max_num_kp2d: 1000 data_dir: - # - ${scan_data_dir}/0606-tiger-others tiger-2 - - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1 + - ${scan_data_dir}/0606-tiger-others tiger-1 + # - ${scan_data_dir}/0620-dinosaurcup-bottle dinosaurcup-1 # - ${scan_data_dir}/demo_bottle_sfm bottle-1 # - ${scan_data_dir}/demo_capture demo-1 From 93bfc80a0760da5eca0d1f20c5c0f7f24ff8a8bb Mon Sep 17 00:00:00 2001 From: Roberto Pellerito Date: Fri, 6 Jan 2023 09:40:21 +0100 Subject: [PATCH 16/16] crop images --- run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.py b/run.py index e6a70fa..92cdd0e 100644 --- a/run.py +++ b/run.py @@ -113,7 +113,7 @@ def sfm(cfg): """Reconstruct and postprocess sparse object point cloud, and store point cloud features""" data_dirs = cfg.dataset.data_dir down_ratio = cfg.sfm.down_ratio - crop_images = False + crop_images = True data_dirs = [data_dirs] if isinstance(data_dirs, str) else data_dirs for data_dir in data_dirs: