diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fc30f7d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.zip filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 29b9942..557eb9e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,13 @@ venv.bak/ # Local outputs logs/ +lsi/models/ +robust_logs/ +reweight_logs/ +truncated_logs/ +label_smoothing_logs/ + +*.pth +*.zip + +RIME/ \ No newline at end of file diff --git a/arm/evaluate.py b/arm/evaluate.py new file mode 100644 index 0000000..ce2e44b --- /dev/null +++ b/arm/evaluate.py @@ -0,0 +1,195 @@ +import numpy as np +import torch +from generate_noise import NoiseGenerator + +def evaluate_grasp( + joint_angles, method, metadata=None, device="cpu", return_features=False +): + # metadata传入模型用来产生measure，还有其他的附加信息 + objs = -np.var(joint_angles, axis=1) + # Remap the objective from [-1, 0] to [0, 100] + objs = (objs + 1.0) * 100.0 + + if metadata is None: + metadata = {} + + cum_theta = np.cumsum(joint_angles, axis=1) + x_pos = np.cos(cum_theta) + y_pos = np.sin(cum_theta) + features = np.concatenate((x_pos, y_pos), axis=1) + + if "dis_embed" in metadata: + if metadata["dis_embed"] is not None: + with torch.no_grad(): + features = ( + metadata["dis_embed"]( + torch.tensor(joint_angles, dtype=torch.float32).to(device) + ) + .detach() + .cpu() + .numpy() + ) + + if method is None: + if return_features: + return objs, features + else: + return objs + elif method in ["qd", "gthf"]: + link_lengths = np.ones(joint_angles.shape[1]) + # theta_1, theta_1 + theta_2, ... + cum_theta = np.cumsum(joint_angles, axis=1) + # l_1 * cos(theta_1), l_2 * cos(theta_1 + theta_2), ... + x_pos = link_lengths[None] * np.cos(cum_theta) + # l_1 * sin(theta_1), l_2 * sin(theta_1 + theta_2), ... + y_pos = link_lengths[None] * np.sin(cum_theta) + + if method == "qd": # 相当于只保留一个二维向量乘以数量，表示“这个 grasp 抓到了哪里” + measures = np.concatenate( + ( + np.sum(x_pos, axis=1, keepdims=True), + np.sum(y_pos, axis=1, keepdims=True), + ), + axis=1, + ) + elif method == "gthf": # 记录了所有关节的位置 + measures = np.concatenate( + ( + np.cumsum(x_pos, axis=1), + np.cumsum(y_pos, axis=1), + ), + axis=1, + ) + elif method == "pca": + assert "pca" in metadata + measures = metadata["pca"].transform(features) + elif method == "ae": + assert "ae" in metadata + with torch.no_grad(): + measures = ( + metadata["ae"](torch.tensor(features, dtype=torch.float32).to(device)) + .detach() + .cpu() + .numpy() + ) + elif method == "qdhf": + measures = features + else: + raise NotImplementedError(f"Unknown method: {method}") + + if return_features: + return objs, measures, features + else: + return objs, measures + + +def replace_sample( + batch_ref_i, batch1_i, batch2_i, model, + device="cpu", K=3, noisy_method=None, parameter=None, itr=0, all_sols=None +): + # ---------- 1. 批量生成 K 个新 p* 与 K 个新 n* ---------- + if itr == 0: + p_expand = torch.tensor( + np.random.uniform(low=-np.pi, high=np.pi, size=(K, 10)), + dtype=torch.float32, + device=device + ) # (K,10) + + n_expand = torch.tensor( + np.random.uniform(low=-np.pi, high=np.pi, size=(K, 10)), + dtype=torch.float32, + device=device + ) + else: + if isinstance(all_sols, np.ndarray): + # all_sols 是 numpy + idx_p = np.random.choice(all_sols.shape[0], size=K, replace=False) + idx_n = np.random.choice(all_sols.shape[0], size=K, replace=False) + p_expand = torch.tensor(all_sols[idx_p], dtype=torch.float32, device=device) + n_expand = torch.tensor(all_sols[idx_n], dtype=torch.float32, device=device) + elif isinstance(all_sols, torch.Tensor): + # all_sols 已经是 torch.Tensor + idx_p = torch.randint(0, all_sols.size(0), size=(K,), device=all_sols.device) + idx_n = torch.randint(0, all_sols.size(0), size=(K,), device=all_sols.device) + p_expand = all_sols[idx_p].to(device).float() # 保证 float32 + n_expand = all_sols[idx_n].to(device).float() + else: + raise TypeError(f"Unsupported type for all_sols: {type(all_sols)}") + + # ---------- 2. 组装 2K 个 (r,p*,n_old) / (r,p_old,n*) ---------- + # 将 r 扩展到 (K,10) + r_tile_p = batch_ref_i.unsqueeze(0).repeat(K, 1) # (K,10) + r_tile_n = r_tile_p.clone() + + # (r, new_p, old_n) + part1 = torch.cat([r_tile_p, p_expand, batch2_i.unsqueeze(0).repeat(K,1)], dim=1) + # (r, old_p, new_n) + part2 = torch.cat([r_tile_n, batch1_i.unsqueeze(0).repeat(K,1), n_expand], dim=1) + + new_batch = torch.cat([part1, part2], dim=0) # (2K, 30) + + # ---------- 3. 计算 ground-truth measure (NumPy) ---------- + # evaluate_grasp 需要形状 (B, 3, 10)；我们这里 batch=2K + new_batch_flat = new_batch.view(2 * K, 3, 10).reshape(6 * K, 10) + new_batch_np = new_batch_flat.cpu().numpy().astype(np.float32) + # print(new_batch_np.shape) + + # 调用 evaluate_grasp；返回 (B, 3, measure_dim) + _, measures = evaluate_grasp( + new_batch_np, # already (2K,3,10) + method="qd", + device=device # evaluate_grasp 内部可能忽略这个参数 + ) + # print(new_gt_measures.shape) + new_gt_measures = measures.reshape(2 * K, 3, 2) + # 取第 0 / 1 / 2 列作为 ref/p/n 的 measure + new_ref_gt = new_gt_measures[:, 0] + new_p_gt = new_gt_measures[:, 1] + new_n_gt = new_gt_measures[:, 2] + + # ---------- 4. 生成干净标签 (+1/-1) ---------- + # new_gt_dis = (new_ref_gt - new_p_gt) - (new_ref_gt - new_n_gt) # (2K,) + new_gt_dis = np.sum( + (np.square(new_ref_gt - new_p_gt) - np.square(new_ref_gt - new_n_gt)), + axis=-1 + ) + + # print(new_gt_dis.shape) + new_lbl_clean = torch.from_numpy((new_gt_dis > 0).astype(np.float32))*2 - 1 # (+1,-1) + new_lbl_clean = new_lbl_clean.to(device) + + # ---------- 5. 加噪（可选） ---------- + noise_gen = NoiseGenerator() + new_lbl_noise = noise_gen.generate_noise( + new_lbl_clean, + new_gt_dis, + noisy_method=noisy_method, + parameter=parameter + ).to(device) # shape: (2K,) + # print(new_lbl_noise.shape) + # ---------- 6. 前向推断 delta_dis（不写入图） ---------- + with torch.no_grad(): + r_all = new_batch[:, :10] # (2K,10) + p_all = new_batch[:, 10:20] + n_all = new_batch[:, 20:30] + + delta_all = model.triplet_delta_dis(r_all, p_all, n_all) # (2K,) + # print(delta_all.shape) + signed = new_lbl_noise * delta_all # (2K,) + + best_idx = torch.argmax(signed) + + # ---------- 7. 返回替换结果 ---------- + if best_idx < K: + # 选择了 part1 → 更新 p + new_p = p_expand[best_idx] + new_n = batch2_i + new_y = new_lbl_noise[best_idx] + else: + # 选择了 part2 → 更新 n + idx = best_idx - K + new_p = batch1_i + new_n = n_expand[idx] + new_y = new_lbl_noise[best_idx] + + return new_p.detach(), new_n.detach(), new_y.detach() diff --git a/arm/generate_noise.py b/arm/generate_noise.py new file mode 100755 index 0000000..c0af89b --- /dev/null +++ b/arm/generate_noise.py @@ -0,0 +1,78 @@ +import torch + + +class NoiseGenerator: + def __init__(self): + pass + + def generate_noise(self, gt, gt_dis, noisy_method, parameter): + if noisy_method == 'stochastic': + return self.stochastic_labels(gt, gt_dis, parameter) + elif noisy_method == 'noisy_labels_exact': + return self.noisy_labels_exact(gt, parameter) + elif noisy_method == 'add_equal_noise': + return self.add_equal_noise(gt, gt_dis, parameter) + elif noisy_method == 'flip_by_distance': + return self.flip_by_distance(gt, gt_dis, parameter) + elif noisy_method == 'flip_labels_asymmetric': + return self.flip_labels_asymmetric(gt, parameter) + else: + raise ValueError(f"Unknown noisy_method: {noisy_method}") + + def noisy_labels_exact(self, gt, noise_rate): + n = len(gt) + n_flip = int(n * noise_rate) + flip_indices = torch.randperm(n)[:n_flip] + gt_noisy = gt.clone() + gt_noisy[flip_indices] = -gt_noisy[flip_indices] + return gt_noisy + + def stochastic_labels(self, gt, gt_dis, noise_ratio): + gt = gt.clone() + n = len(gt) + n_noise = int(n * noise_ratio) + + noise_indices = torch.randperm(n)[:n_noise] + + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + prob = torch.sigmoid(-gt_dis_tensor[noise_indices]) + sampled = torch.bernoulli(prob) + noisy_labels = sampled * 2 - 1 + + gt[noise_indices] = noisy_labels + return gt + + def add_equal_noise(self, gt, gt_dis, delta_equal): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + equal_mask = torch.abs(gt_dis_tensor) < delta_equal + + gt_noisy = gt.clone() + gt_noisy[equal_mask] = 0 + return gt_noisy + + def flip_by_distance(self, gt, gt_dis, delta_threshold): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + flip_mask = torch.abs(gt_dis_tensor) < delta_threshold + + gt_noisy = gt.clone() + gt_noisy[flip_mask] = -gt_noisy[flip_mask] + return gt_noisy + + def flip_labels_asymmetric(self, gt, flip_rate=0.1): + flip_rate_neg = flip_rate + flip_rate_pos = flip_rate + gt = gt.clone() + pos_indices = torch.where(gt == 1)[0] + neg_indices = torch.where(gt == -1)[0] + + n_pos_flip = int(len(pos_indices) * flip_rate_pos) + n_neg_flip = int(len(neg_indices) * flip_rate_neg) + + pos_flip_idx = pos_indices[torch.randperm(len(pos_indices))[ + :n_pos_flip]] + neg_flip_idx = neg_indices[torch.randperm(len(neg_indices))[ + :n_neg_flip]] + + gt[pos_flip_idx] = -1 + gt[neg_flip_idx] = 1 + return gt diff --git a/arm/main.py b/arm/main.py old mode 100644 new mode 100755 index 2ff31c0..78a30a5 --- a/arm/main.py +++ b/arm/main.py @@ -6,7 +6,7 @@ import sys import time from pathlib import Path - +import argparse import matplotlib import numpy as np import torch @@ -22,10 +22,13 @@ sys.path.append(".") +# dis_embed是对比学习的latent projector +# 最终行为空间是二维的，可以画 heatmap def evaluate_grasp( joint_angles, method, metadata=None, device="cpu", return_features=False ): + # metadata传入模型用来产生measure，还有其他的附加信息 objs = -np.var(joint_angles, axis=1) # Remap the objective from [-1, 0] to [0, 100] objs = (objs + 1.0) * 100.0 @@ -64,7 +67,7 @@ def evaluate_grasp( # l_1 * sin(theta_1), l_2 * sin(theta_1 + theta_2), ... y_pos = link_lengths[None] * np.sin(cum_theta) - if method == "qd": + if method == "qd": # 相当于只保留一个二维向量乘以数量，表示“这个 grasp 抓到了哪里” measures = np.concatenate( ( np.sum(x_pos, axis=1, keepdims=True), @@ -72,7 +75,7 @@ def evaluate_grasp( ), axis=1, ) - elif method == "gthf": + elif method == "gthf": # 记录了所有关节的位置 measures = np.concatenate( ( np.cumsum(x_pos, axis=1), @@ -121,7 +124,7 @@ def create_optimizer( if method == "qd": assert gt_bounds is not None - objs, measures = evaluate_grasp(sols, method, metadata) + objs, measures = evaluate_grasp(sols, method, metadata, device) if archive_bounds is None: archive_bounds = gt_bounds elif method == "pca": @@ -147,6 +150,8 @@ def create_optimizer( else: raise NotImplementedError(f"Unknown method: {method}") + # 得到measure更新archive_bounds + archive = GridArchive((50, 50), archive_bounds, seed) archive.initialize(solution_dim=len(sols[0])) # Add each solution to the archive. @@ -208,27 +213,31 @@ def save_heatmap(archive, heatmap_path): def run_experiment( method, trial_id, - dim=1000, - init_pop=1000, - itrs=10000, + dim=10, + init_pop=100, + itrs=10000, outdir="logs", log_freq=1, - log_arch_freq=1000, + log_arch_freq=1000, # log frequency seed=None, use_dis_embed=False, - n_pref_data=1000, + n_pref_data=1000, # number of preference data online_finetune=False, incre_bounds=False, + noisy_method=None, + parameter=None, + robust_loss=None, + device='cpu' ): algorithm = "map_elites" - device = "cpu" if seed is not None: np.random.seed(seed) torch.manual_seed(seed) # Create a directory for this specific trial. - s_logdir = os.path.join(outdir, f"{algorithm}_trial_{trial_id}") + num_parameter = str(100* parameter) + s_logdir = os.path.join(outdir, f"{noisy_method}_{num_parameter}_trial{trial_id}_{seed}") logdir = Path(s_logdir) if not logdir.is_dir(): logdir.mkdir() @@ -255,10 +264,10 @@ def run_experiment( "Maximum", "Average", "QD-Score (search)", - "Coverage (search)", + "Coverage (search)", # "search" 指的是 搜索过程中的所有被探索过的解 "Maximum (search)", "Average (search)", - "QD-Score (fit)", + "QD-Score (fit)", # "fit" 指的是 最终 archive 中保留下来的精英解 "Coverage (fit)", "Maximum (fit)", "Average (fit)", @@ -299,7 +308,7 @@ def run_experiment( inputs = np.random.uniform( low=-np.pi, high=np.pi, size=(n_pref_data * 3, dim) ) - _, gt_measures = evaluate_grasp(inputs, method="qd") + _, gt_measures = evaluate_grasp(inputs, method="qd", device=device) # gt_measure就是机械臂末端位置 dis_embed_data = inputs.reshape((n_pref_data, 3, dim)) dis_embed_gt_measures = gt_measures.reshape((n_pref_data, 3, 2)) dis_embed, dis_embed_acc = fit_dis_embed( @@ -307,6 +316,11 @@ def run_experiment( dis_embed_gt_measures, latent_dim=2, seed=seed, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss, + device=device, + itr=itr ) else: dis_embed = None @@ -317,7 +331,7 @@ def run_experiment( # sols = np.concatenate((sols, np.array(outliers)), axis=0) # Update the dis embed. - if use_dis_embed: + if use_dis_embed: # 使用archive里新的sol训练 加噪声影响这个过程吗 additional_inputs = [ all_sols[np.random.choice(all_sols.shape[0], 3)] for _ in range(n_pref_data) @@ -325,7 +339,7 @@ def run_experiment( additional_inputs = np.array(additional_inputs) _, additional_gt_measures = evaluate_grasp( additional_inputs.reshape(n_pref_data * 3, dim), - method="qd", + method="qd", device=device ) additional_gt_measures = additional_gt_measures.reshape( n_pref_data, 3, 2 @@ -341,7 +355,14 @@ def run_experiment( dis_embed_gt_measures, latent_dim=2, seed=seed, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss, + device=device, + itr=itr, + all_sols=all_sols ) + # model, acc metadata = {"dis_embed": dis_embed} _, all_features = evaluate_grasp( @@ -359,6 +380,7 @@ def run_experiment( ae = fit_ae(all_features, device=device) metadata["ae"] = ae + # 如果需要更新archive，包括optimizer在内都是重新初始化 archive, optimizer, metadata = create_optimizer( method, all_sols, @@ -369,19 +391,21 @@ def run_experiment( gt_bounds=gt_archive_bounds, seed=seed, ) + # Optimizer(archive,emitters,init_archive=False) archive_bounds = metadata["archive_bounds"] - _objs, _gt_measures = evaluate_grasp(all_sols, method="qd") + _objs, _gt_measures = evaluate_grasp(all_sols, method="qd", device=device) for i in range(len(all_sols)): gt_archive_all.add(all_sols[i], _objs[i], _gt_measures[i]) - sols = optimizer.ask() + # 对于普通的步骤，只需要更新solution和obj + sols = optimizer.ask() # 让emitters生成新的解 objs, measures, features = evaluate_grasp( sols, method, metadata, device, return_features=True ) best = max(best, max(objs)) - _objs, _gt_measures = evaluate_grasp(sols, method="qd") + _objs, _gt_measures = evaluate_grasp(sols, method="qd", device=device) for i in range(len(sols)): gt_archive_all.add(sols[i], _objs[i], _gt_measures[i]) @@ -402,7 +426,7 @@ def run_experiment( archive_bounds[1, 1] = np.max(measures[:, 1]) update_archive = True - if update_archive: + if update_archive: # 如果更新了bound， all_sols = archive.data()[0] all_sols = np.concatenate((all_sols, sols), axis=0) _, all_features = evaluate_grasp( @@ -412,6 +436,7 @@ def run_experiment( device=device, return_features=True, ) + # 只返回所有solution的关节坐标 archive, optimizer, metadata = create_optimizer( method, all_sols, @@ -438,7 +463,7 @@ def run_experiment( gt_archive = GridArchive((50, 50), gt_archive_bounds, seed=seed) gt_archive.initialize(dim) sols = archive.data()[0] - objs, gt_measures = evaluate_grasp(sols, method="qd") + objs, gt_measures = evaluate_grasp(sols, method="qd",device=device) for i in range(len(sols)): gt_archive.add(sols[i], objs[i], gt_measures[i]) @@ -510,17 +535,22 @@ def run_experiment( def arm_main( method, - trial_id=0, + trial_id=1, dim=10, init_pop=100, itrs=1000, - outdir="logs", + outdir='logs', log_freq=20, log_arch_freq=100, use_dis_embed=False, n_pref_data=1000, online_finetune=False, incre_bounds=False, + noisy_method=None, + parameter=None, + seed=None, + robust_loss=None, + device='cpu' ): """Experimental tool for the planar robotic arm experiments.""" @@ -539,49 +569,91 @@ def arm_main( outdir=outdir, log_freq=log_freq, log_arch_freq=log_arch_freq, - seed=trial_id, + seed=seed, use_dis_embed=use_dis_embed, n_pref_data=n_pref_data, online_finetune=online_finetune, incre_bounds=incre_bounds, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss, + device=device ) if __name__ == "__main__": - if len(sys.argv) > 1: - trial_id = int(sys.argv[1]) - else: - trial_id = 0 - + # if len(sys.argv) > 1: + # trial_id = int(sys.argv[1]) + # else: + # trial_id = 0 + + parser = argparse.ArgumentParser( + description="Run robot arm experiments") + parser.add_argument('--trial_id', type=int, default=0) + parser.add_argument('--noisy_method', type=str, choices=['stochastic', 'add_equal_noise', + 'flip_by_distance', 'flip_labels_asymmetric', 'noisy_labels_exact'], required=True) + parser.add_argument('--parameter', type=float, required=True) + parser.add_argument('--robust_loss',type=str,required=True) + parser.add_argument('--seed', type=int, required=True) + parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') + parser.add_argument('--cuda_index', type=int, default=0) + + args = parser.parse_args() # QD-GT - arm_main(method="qd", trial_id=trial_id) + # arm_main(method="qd", trial_id=trial_id) # AURORA - for online_finetune in [False, True]: - arm_main( - method="pca", - trial_id=trial_id, - online_finetune=online_finetune, - ) - arm_main( - method="ae", - trial_id=trial_id, - online_finetune=online_finetune, - ) + # for online_finetune in [False, True]: + # arm_main( + # method="pca", + # trial_id=trial_id, + # online_finetune=online_finetune, + # ) + # arm_main( + # method="ae", + # trial_id=trial_id, + # online_finetune=online_finetune, + # ) + + if args.device == 'cuda': + if torch.cuda.is_available(): + device = torch.device(f'cuda:{args.cuda_index}') + else: + print("⚠️ CUDA was requested but is not available. Falling back to CPU.") + device = torch.device('cpu') + else: + device = torch.device('cpu') # QDHF n_pref_data = 1000 + out_dir = f'logs/{args.robust_loss}_logs' + os.makedirs(out_dir, exist_ok=True) + # arm_main( + # method="qdhf", + # trial_id=args.trial_id, + # use_dis_embed=True, + # n_pref_data=n_pref_data, + # online_finetune=False, + # noisy_method=args.noisy_method, + # parameter=args.parameter, + # seed=args.seed + # ) arm_main( method="qdhf", - trial_id=trial_id, - use_dis_embed=True, - n_pref_data=n_pref_data, - online_finetune=False, - ) - arm_main( - method="qdhf", - trial_id=trial_id, + trial_id=args.trial_id, + outdir=out_dir, use_dis_embed=True, n_pref_data=n_pref_data // 4, online_finetune=True, + noisy_method=args.noisy_method, + parameter=args.parameter, + seed=args.seed, + robust_loss=args.robust_loss, + device=device ) + + if args.device == 'cuda': + torch.cuda.empty_cache() + gc.collect() + + sys.exit(0) diff --git a/arm/pic/coverage_flip_labels_asymmetric.png b/arm/pic/coverage_flip_labels_asymmetric.png new file mode 100644 index 0000000..11a08a8 Binary files /dev/null and b/arm/pic/coverage_flip_labels_asymmetric.png differ diff --git a/arm/pic/coverage_noisy_labels_exact.png b/arm/pic/coverage_noisy_labels_exact.png new file mode 100644 index 0000000..f6e5abc Binary files /dev/null and b/arm/pic/coverage_noisy_labels_exact.png differ diff --git a/arm/pic/coverage_stochastic.png b/arm/pic/coverage_stochastic.png new file mode 100644 index 0000000..d280959 Binary files /dev/null and b/arm/pic/coverage_stochastic.png differ diff --git a/arm/pic/qd_score_flip_labels_asymmetric.png b/arm/pic/qd_score_flip_labels_asymmetric.png new file mode 100644 index 0000000..d723ac3 Binary files /dev/null and b/arm/pic/qd_score_flip_labels_asymmetric.png differ diff --git a/arm/pic/qd_score_noisy_labels_exact.png b/arm/pic/qd_score_noisy_labels_exact.png new file mode 100644 index 0000000..acb017d Binary files /dev/null and b/arm/pic/qd_score_noisy_labels_exact.png differ diff --git a/arm/pic/qd_score_stochastic.png b/arm/pic/qd_score_stochastic.png new file mode 100644 index 0000000..968b2ac Binary files /dev/null and b/arm/pic/qd_score_stochastic.png differ diff --git a/arm/ribs/__init__.py b/arm/ribs/__init__.py old mode 100644 new mode 100755 diff --git a/arm/ribs/archives/__init__.py b/arm/ribs/archives/__init__.py old mode 100644 new mode 100755 index a9cf205..bc2b66f --- a/arm/ribs/archives/__init__.py +++ b/arm/ribs/archives/__init__.py @@ -26,3 +26,5 @@ "ArchiveBase", "AddStatus", ] + +# 从外部调用 from ribs.archives import * 会只导入上述五个类。 \ No newline at end of file diff --git a/arm/ribs/archives/_add_status.py b/arm/ribs/archives/_add_status.py old mode 100644 new mode 100755 diff --git a/arm/ribs/archives/_archive_base.py b/arm/ribs/archives/_archive_base.py old mode 100644 new mode 100755 index a3be3a8..9834e3d --- a/arm/ribs/archives/_archive_base.py +++ b/arm/ribs/archives/_archive_base.py @@ -43,7 +43,8 @@ def __init__(self, seed=None, buf_size=10): self._rng = np.random.default_rng(seed) self._buf_size = buf_size - self._buffer = self._rng.random(buf_size) + self._buffer = self._rng.random(buf_size) + # 一次性生成 buf_size 个 [0, 1) 区间内的随机浮点数，存入 self._buffer self._buf_idx = 0 def get(self, max_val): @@ -55,6 +56,7 @@ def get(self, max_val): if self._buf_idx >= self._buf_size: self._buf_idx = 0 self._buffer = self._rng.random(self._buf_size) + # 刷新整个 buffer return val @@ -71,6 +73,8 @@ class ArchiveBase(ABC): # pylint: disable = too-many-instance-attributes common dimensions. Using the ``storage_dims`` and ``behavior_dim`` arguments in :meth:`__init__` and the ``solution_dim`` argument in ``initialize``, these arrays are as follows: + + 是否被占用 解本身 目标值 行为值 元数据 +------------------------+------------------------------------+ | Name | Shape | @@ -164,10 +168,10 @@ def __init__(self, storage_dims, behavior_dim, seed=None, dtype=np.float64): # docstring). self._rand_buf = None self._seed = seed - self._initialized = False + self._initialized = False # 是否初始化 self._bins = np.product(self._storage_dims) - self._dtype = self._parse_dtype(dtype) + self._dtype = self._parse_dtype(dtype) # 定义浮点数精度 @staticmethod def _parse_dtype(dtype): diff --git a/arm/ribs/archives/_cvt_archive.py b/arm/ribs/archives/_cvt_archive.py old mode 100644 new mode 100755 diff --git a/arm/ribs/archives/_grid_archive.py b/arm/ribs/archives/_grid_archive.py old mode 100644 new mode 100755 index 9af63eb..e786d71 --- a/arm/ribs/archives/_grid_archive.py +++ b/arm/ribs/archives/_grid_archive.py @@ -39,6 +39,8 @@ class GridArchive(ArchiveBase): def __init__(self, dims, ranges, seed=None, dtype=np.float64): self._dims = np.array(dims) + # dims = [20, 30, 40]三个维度 + # ranges [(-1, 1), (-2, 2)] if len(self._dims) != len(ranges): raise ValueError(f"dims (length {len(self._dims)}) and ranges " f"(length {len(ranges)}) must be the same length") @@ -51,10 +53,11 @@ def __init__(self, dims, ranges, seed=None, dtype=np.float64): dtype=dtype, ) - ranges = list(zip(*ranges)) + ranges = list(zip(*ranges)) + # self._boundaries 是一个包含两个 numpy.ndarray 的列表，每个 numpy.ndarray 有 51 个边界值 self._lower_bounds = np.array(ranges[0], dtype=self.dtype) self._upper_bounds = np.array(ranges[1], dtype=self.dtype) - self._interval_size = self._upper_bounds - self._lower_bounds + self._interval_size = self._upper_bounds - self._lower_bounds # bin的大小 self._boundaries = [] for dim, lower_bound, upper_bound in zip(self._dims, self._lower_bounds, @@ -115,7 +118,7 @@ def _get_index_numba(behavior_values, upper_bounds, lower_bounds, # the grid. behavior_values = np.minimum( np.maximum(behavior_values + _EPSILON, lower_bounds), - upper_bounds - _EPSILON) + upper_bounds - _EPSILON) # 防止超出范围 index = (behavior_values - lower_bounds) / interval_size * dims return index.astype(np.int32) diff --git a/arm/ribs/archives/_sliding_boundaries_archive.py b/arm/ribs/archives/_sliding_boundaries_archive.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/__init__.py b/arm/ribs/emitters/__init__.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_emitter_base.py b/arm/ribs/emitters/_emitter_base.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_gaussian_emitter.py b/arm/ribs/emitters/_gaussian_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_gradient_emitter.py b/arm/ribs/emitters/_gradient_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_gradient_improvement_emitter.py b/arm/ribs/emitters/_gradient_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_improvement_emitter.py b/arm/ribs/emitters/_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_iso_line_emitter.py b/arm/ribs/emitters/_iso_line_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_optimizing_emitter.py b/arm/ribs/emitters/_optimizing_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/_random_direction_emitter.py b/arm/ribs/emitters/_random_direction_emitter.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/opt/__init__.py b/arm/ribs/emitters/opt/__init__.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/opt/_adam.py b/arm/ribs/emitters/opt/_adam.py old mode 100644 new mode 100755 diff --git a/arm/ribs/emitters/opt/_cma_es.py b/arm/ribs/emitters/opt/_cma_es.py old mode 100644 new mode 100755 diff --git a/arm/ribs/factory.py b/arm/ribs/factory.py old mode 100644 new mode 100755 diff --git a/arm/ribs/optimizers/__init__.py b/arm/ribs/optimizers/__init__.py old mode 100644 new mode 100755 diff --git a/arm/ribs/optimizers/_optimizer.py b/arm/ribs/optimizers/_optimizer.py old mode 100644 new mode 100755 diff --git a/arm/ribs/visualize.py b/arm/ribs/visualize.py old mode 100644 new mode 100755 diff --git a/arm/robust_loss.py b/arm/robust_loss.py new file mode 100644 index 0000000..423d9a9 --- /dev/null +++ b/arm/robust_loss.py @@ -0,0 +1,168 @@ +import torch +import torch.nn as nn + +class RobustLossAgent: + def __init__(self, margin=0.05): + self.margin = margin + + def robust_loss(self, delta_dis, y, robust_loss, parameter, robust_parameter=None, epoch=0): + robust_parameter = robust_parameter or {} + + if robust_loss == 'reweight': + beta = 0.2 + return self.reweighted_triplet_loss(delta_dis, y, beta=beta) + + elif robust_loss == 'truncated': + alpha = 0.05 + epsilon_max = 0.2 + return self.truncated_triplet_loss(delta_dis, y, epoch, alpha=alpha, epsilon_max=epsilon_max) + + elif robust_loss == 'label_smoothing': + alpha = 0.05 + return self.label_smoothing_triplet_loss(delta_dis, y, alpha=alpha) + + elif robust_loss == 'rDPO': + epsilon = parameter + return self.rDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'cDPO': + epsilon = parameter + return self.cDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'None': + loss_fn = lambda y, delta_dis: torch.max(torch.tensor([0.0], device=y.device), 0.05 - y * delta_dis).mean() + return loss_fn(y, delta_dis) + + elif robust_loss == 'robust_qdhf': + # beta = 0.2 + # return self.reweighted_triplet_loss(delta_dis, y, beta=beta) + loss_fn = lambda y, delta_dis: torch.max(torch.tensor([0.0], device=y.device), 0.05 - y * delta_dis).mean() + return loss_fn(y, delta_dis) + else: + raise ValueError(f"unknown robust method: {robust_loss}") + + def reweighted_triplet_loss(self, delta_dis, y, beta=0.2, eps=1e-6): + pred = torch.sigmoid(delta_dis) + weight = pred ** beta + raw_loss = weight * torch.clamp(self.margin - y * delta_dis, min=0.0) + normalized_loss = raw_loss.sum() * (delta_dis.size(0) / (weight.sum() + eps)) + return normalized_loss + + def truncated_triplet_loss(self, delta_dis, y, epoch, alpha=0.1, epsilon_max=0.1): + loss_sample = torch.clamp(self.margin - y * delta_dis, min=0.0) + drop_rate = min(alpha * epoch, epsilon_max) + batch_size = loss_sample.size(0) + + if drop_rate > 0: + kth = int(batch_size * (1 - drop_rate)) + if kth <= 0: + loss_sample = torch.zeros_like(loss_sample) + else: + tau = torch.kthvalue(loss_sample, kth).values + keep_mask = (loss_sample <= tau) + loss_sample = loss_sample * keep_mask.float() + + return loss_sample.mean() + + + def smooth_labels(self, y, alpha=0.1, num_classes=2): + if alpha <= 1e-6: + return y, y # 不平滑 + + y_idx = ((y + 1) / 2).long() + I = torch.eye(num_classes, device=y.device, dtype=y.dtype) + J = torch.ones((num_classes, num_classes), device=y.device, dtype=y.dtype) + M = (1 - alpha) * I + (alpha / num_classes) * J + + soft_labels = M[y_idx] + y_smooth = soft_labels[:, 1] - soft_labels[:, 0] # 转换为 ∈ [-1, 1] + y_smooth = torch.clamp(y_smooth, -1.0, 1.0) + return soft_labels, y_smooth + + + def label_smoothing_triplet_loss(self, delta_dis, y, alpha): + _, y_smooth = self.smooth_labels(y, alpha=alpha, num_classes=2) + loss = torch.clamp(self.margin - y_smooth * delta_dis, min=0.0) + return loss.mean() + + + def rDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + # print(epsilon) + y = y.to(delta_dis.device) + loss_clean = -torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + loss_flipped = -torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + numerator = (1 - epsilon) * loss_clean - epsilon * loss_flipped + denominator = 1 - 2 * epsilon + corrected_loss = numerator / (denominator + 1e-8) # add epsilon to avoid div 0 + return corrected_loss.mean() + + def cDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + y = y.to(delta_dis.device) + log_prob_clean = torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + log_prob_flipped = torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + # Weighted sum per noisy label distribution + loss = -((1 - epsilon) * log_prob_clean + epsilon * log_prob_flipped) + return loss.mean() + + + +class TripletCDRP(nn.Module): + def __init__(self, gamma=5.0, eta=0.5, eps=0.01): + super(TripletCDRP, self).__init__() + self.gamma = gamma + self.eta = eta + self.eps = eps + + def forward(self, ref_embed, x1_embed, x2_embed, pref_label): + d1 = torch.sum((ref_embed - x1_embed) ** 2, dim=-1) + d2 = torch.sum((ref_embed - x2_embed) ** 2, dim=-1) + logits = torch.stack([-d1, -d2], dim=1) # (B, 2) + logits = torch.clamp(logits, min=-20, max=20) + probs = torch.softmax(logits, dim=1) # (B, 2) + + labels = ((pref_label + 1) // 2).long() + + # Base CE loss + ce_loss = nn.NLLLoss(reduction="none") + log_probs = torch.log(probs + 1e-8) + # nominal_loss = ce_loss(log_probs, labels) # (B,) + + # C(y', y) = gamma if y' != y else 0 + gamma = self.gamma + penalty = gamma * (1 - torch.nn.functional.one_hot(labels, num_classes=2).to(logits.device)) + + # robust loss inner: max_y' [l(y') - gamma * 1_{y' ≠ y}] + loss_all = ce_loss(log_probs.repeat(1, 2).reshape(-1, 2), torch.tensor([0, 1] * logits.shape[0]).to(logits.device)).reshape(-1, 2) + robust_loss = (loss_all - penalty).max(dim=1).values + + final_loss = self.eps * gamma + robust_loss.mean() + + return final_loss + + +class InstanceEarlyStopper: + def __init__(self, num_samples, patience=3, delta=1e-4): + self.patience = patience + self.delta = delta + self.loss_history = [[] for _ in range(num_samples)] + self.mask = torch.zeros(num_samples, dtype=torch.bool) # True 表示“停止训练该样本” + + def update(self, indices, losses): + for i, idx in enumerate(indices): + idx = int(idx) + if self.mask[idx]: + continue + self.loss_history[idx].append(losses[i].item()) + + if len(self.loss_history[idx]) >= self.patience: + l_hist = self.loss_history[idx][-self.patience:] + if len(l_hist) == self.patience: + delta2 = sum([-l_hist[j] + 2*l_hist[j+1] - l_hist[j+2] for j in range(self.patience - 2)]) + + if abs(delta2) < self.delta: + self.mask[idx] = True + + def get_active_indices(self, all_indices): + return [idx for idx in all_indices if not self.mask[int(idx)]] diff --git a/arm/run_arm.sh b/arm/run_arm.sh new file mode 100755 index 0000000..2b52f80 --- /dev/null +++ b/arm/run_arm.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export CUDA_VISIBLE_DEVICES=0 + +python run_experiment2.py --noise_type noisy_labels_exact --robust cDPO --device cuda +# noisy_labels_exact stochastic flip_labels_asymmetric +# source .venv/bin/activate \ No newline at end of file diff --git a/arm/run_experiment.py b/arm/run_experiment.py new file mode 100755 index 0000000..e4131f2 --- /dev/null +++ b/arm/run_experiment.py @@ -0,0 +1,120 @@ +import subprocess +import pandas as pd +import numpy as np +import re +import os +import gc +import time +import torch +import argparse + +parser = argparse.ArgumentParser(description="Run QDHF experiments with config") +parser.add_argument("--noise_type", type=str, required=True) +parser.add_argument("--robust", type=str) +parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') +args = parser.parse_args() + +# 实验设置 +noisy_list = { + "noisy_labels_exact": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "stochastic": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "add_equal_noise": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_by_distance": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_labels_asymmetric": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +} +noisy_methods = {args.noise_type: noisy_list[args.noise_type]} +base_trial_ids = { + "stochastic": 10, + "add_equal_noise": 20, + "flip_by_distance": 30, + "flip_labels_asymmetric": 40, + "noisy_labels_exact": 50 +} +seeds = ["1111", "2222", "3333", "4444", "5555", "6666", "7777", "8888", "9999", "42"] + +# 主逻辑 +for method, params in noisy_methods.items(): + csv_path = f"/mnt/data6t/qdhf/arm/logs/{args.robust}_logs/{method}_experiment_results.csv" + + # 检查文件是否存在，如果存在则跳过创建 + if os.path.exists(csv_path): + print(f"CSV file for method {method} already exists, skipping creation.") + header_written = False + else: + header_written = True + + base_trial_id = base_trial_ids[method] + for i, param in enumerate(params): + trial_id = base_trial_id + i + + qd_scores = [] + coverage_scores = [] + + + for seed in seeds: + print(f"Running experiment with method={method}, param={param}, robust={args.robust}, trial_id={trial_id}, seed={seed}") + result = subprocess.run( + ["python", "main.py", "--seed", seed, "--trial_id", str(trial_id), + "--noisy_method", method, "--parameter", str(param), "--robust_loss", + args.robust, "--device", args.device], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + result.check_returncode() + output = result.stdout + print(output) + + try: + match = re.search(r"QD score: ([\d\.]+) Coverage: ([\d\.]+)", output) + if match: + qd_score = float(match.group(1)) + coverage = float(match.group(2)) + else: + qd_score, coverage = np.nan, np.nan + except Exception: + qd_score, coverage = np.nan, np.nan + + qd_scores.append(qd_score) + coverage_scores.append(coverage) + + # 每次实验结果写入 CSV + record = { + "Method": method, + "Parameter": param, + "Seed": seed, + "Trial ID": trial_id, + "QD Score": qd_score, + "Coverage": coverage + } + + pd.DataFrame([record]).to_csv(csv_path, mode='a', header=header_written, index=False) + header_written = False # 只有第一次写入时才写表头 + + gc.collect() + time.sleep(2) + + qd_mean = np.nanmean(qd_scores) + qd_std = np.nanstd(qd_scores) + coverage_mean = np.nanmean(coverage_scores) + coverage_std = np.nanstd(coverage_scores) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "Mean", + "Trial ID": "Mean", + "QD Score": qd_mean, + "Coverage": coverage_mean + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "STD", + "Trial ID": "STD", + "QD Score": qd_std, + "Coverage": coverage_std + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + +print(f"Experiment results have been saved to {csv_path}") diff --git a/arm/run_experiment1.py b/arm/run_experiment1.py new file mode 100644 index 0000000..b92e1cc --- /dev/null +++ b/arm/run_experiment1.py @@ -0,0 +1,120 @@ +import subprocess +import pandas as pd +import numpy as np +import re +import os +import gc +import time +import torch +import argparse + +parser = argparse.ArgumentParser(description="Run QDHF experiments with config") +parser.add_argument("--robust", type=str) +parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') +args = parser.parse_args() + +device = 'cuda' +robust = 'crdo' +# 实验设置 +noisy_methods = { + # "noisy_labels_exact": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] + # "stochastic": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + # "add_equal_noise": [1, 2, 5, 10, 15, 20, 25, 30], + # "flip_by_distance": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_labels_asymmetric": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +} +base_trial_ids = { + "stochastic": 10, + "add_equal_noise": 20, + "flip_by_distance": 30, + "flip_labels_asymmetric": 40, + "noisy_labels_exact": 50 +} +seeds = ["1111", "2222", "3333", "4444", "5555", "6666", "7777", "8888", "9999", "42"] + +# 主逻辑 +for method, params in noisy_methods.items(): + csv_path = f"/mnt/data6t/qdhf/arm/{args.robust}_logs/{method}_experiment_results.csv" + + # 检查文件是否存在，如果存在则跳过创建 + if os.path.exists(csv_path): + print(f"CSV file for method {method} already exists, skipping creation.") + header_written = False + else: + header_written = True + + base_trial_id = base_trial_ids[method] + for i, param in enumerate(params): + trial_id = base_trial_id + i + + qd_scores = [] + coverage_scores = [] + + + for seed in seeds: + print(f"Running experiment with method={method}, param={param}, robust={args.robust}, trial_id={trial_id}, seed={seed}") + result = subprocess.run( + ["python", "main.py", "--seed", seed, "--trial_id", str(trial_id), + "--noisy_method", method, "--parameter", str(param), "--robust_loss", + robust, "--device", device], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + result.check_returncode() + output = result.stdout + print(output) + + try: + match = re.search(r"QD score: ([\d\.]+) Coverage: ([\d\.]+)", output) + if match: + qd_score = float(match.group(1)) + coverage = float(match.group(2)) + else: + qd_score, coverage = np.nan, np.nan + except Exception: + qd_score, coverage = np.nan, np.nan + + qd_scores.append(qd_score) + coverage_scores.append(coverage) + + # 每次实验结果写入 CSV + record = { + "Method": method, + "Parameter": param, + "Seed": seed, + "Trial ID": trial_id, + "QD Score": qd_score, + "Coverage": coverage + } + + pd.DataFrame([record]).to_csv(csv_path, mode='a', header=header_written, index=False) + header_written = False # 只有第一次写入时才写表头 + + gc.collect() + time.sleep(2) + + qd_mean = np.nanmean(qd_scores) + qd_std = np.nanstd(qd_scores) + coverage_mean = np.nanmean(coverage_scores) + coverage_std = np.nanstd(coverage_scores) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "Mean", + "Trial ID": "Mean", + "QD Score": qd_mean, + "Coverage": coverage_mean + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "STD", + "Trial ID": "STD", + "QD Score": qd_std, + "Coverage": coverage_std + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + +print(f"Experiment results have been saved to {csv_path}") diff --git a/arm/run_experiment2.py b/arm/run_experiment2.py new file mode 100644 index 0000000..e4131f2 --- /dev/null +++ b/arm/run_experiment2.py @@ -0,0 +1,120 @@ +import subprocess +import pandas as pd +import numpy as np +import re +import os +import gc +import time +import torch +import argparse + +parser = argparse.ArgumentParser(description="Run QDHF experiments with config") +parser.add_argument("--noise_type", type=str, required=True) +parser.add_argument("--robust", type=str) +parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') +args = parser.parse_args() + +# 实验设置 +noisy_list = { + "noisy_labels_exact": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "stochastic": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "add_equal_noise": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_by_distance": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_labels_asymmetric": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +} +noisy_methods = {args.noise_type: noisy_list[args.noise_type]} +base_trial_ids = { + "stochastic": 10, + "add_equal_noise": 20, + "flip_by_distance": 30, + "flip_labels_asymmetric": 40, + "noisy_labels_exact": 50 +} +seeds = ["1111", "2222", "3333", "4444", "5555", "6666", "7777", "8888", "9999", "42"] + +# 主逻辑 +for method, params in noisy_methods.items(): + csv_path = f"/mnt/data6t/qdhf/arm/logs/{args.robust}_logs/{method}_experiment_results.csv" + + # 检查文件是否存在，如果存在则跳过创建 + if os.path.exists(csv_path): + print(f"CSV file for method {method} already exists, skipping creation.") + header_written = False + else: + header_written = True + + base_trial_id = base_trial_ids[method] + for i, param in enumerate(params): + trial_id = base_trial_id + i + + qd_scores = [] + coverage_scores = [] + + + for seed in seeds: + print(f"Running experiment with method={method}, param={param}, robust={args.robust}, trial_id={trial_id}, seed={seed}") + result = subprocess.run( + ["python", "main.py", "--seed", seed, "--trial_id", str(trial_id), + "--noisy_method", method, "--parameter", str(param), "--robust_loss", + args.robust, "--device", args.device], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + result.check_returncode() + output = result.stdout + print(output) + + try: + match = re.search(r"QD score: ([\d\.]+) Coverage: ([\d\.]+)", output) + if match: + qd_score = float(match.group(1)) + coverage = float(match.group(2)) + else: + qd_score, coverage = np.nan, np.nan + except Exception: + qd_score, coverage = np.nan, np.nan + + qd_scores.append(qd_score) + coverage_scores.append(coverage) + + # 每次实验结果写入 CSV + record = { + "Method": method, + "Parameter": param, + "Seed": seed, + "Trial ID": trial_id, + "QD Score": qd_score, + "Coverage": coverage + } + + pd.DataFrame([record]).to_csv(csv_path, mode='a', header=header_written, index=False) + header_written = False # 只有第一次写入时才写表头 + + gc.collect() + time.sleep(2) + + qd_mean = np.nanmean(qd_scores) + qd_std = np.nanstd(qd_scores) + coverage_mean = np.nanmean(coverage_scores) + coverage_std = np.nanstd(coverage_scores) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "Mean", + "Trial ID": "Mean", + "QD Score": qd_mean, + "Coverage": coverage_mean + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "STD", + "Trial ID": "STD", + "QD Score": qd_std, + "Coverage": coverage_std + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + +print(f"Experiment results have been saved to {csv_path}") diff --git a/arm/run_simple.sh b/arm/run_simple.sh new file mode 100755 index 0000000..6f207ce --- /dev/null +++ b/arm/run_simple.sh @@ -0,0 +1,7 @@ +#!/bin/bash + + +# python run_experiment1.py +# python main.py --seed 1111 --trial_id 53 --noisy_method noisy_labels_exact --parameter 0.3 --robust_loss label_smoothing +python main.py --seed 2222 --trial_id 53 --noisy_method noisy_labels_exact --parameter 0.3 --robust_loss robust_qdhf --device cuda --cuda_index 4 +# python main.py --seed 5555 --trial_id 53 --noisy_method noisy_labels_exact --parameter 0.3 --robust_loss rDPO \ No newline at end of file diff --git a/arm/utils.py b/arm/utils.py old mode 100644 new mode 100755 index f030597..5a7f6f0 --- a/arm/utils.py +++ b/arm/utils.py @@ -1,9 +1,12 @@ from sklearn.decomposition import PCA import torch +import copy from torch import nn import numpy as np import time - +from generate_noise import NoiseGenerator +from robust_loss import RobustLossAgent, TripletCDRP, InstanceEarlyStopper +from evaluate import replace_sample class DisEmbed(nn.Module): def __init__(self, input_dim, latent_dim): @@ -38,15 +41,22 @@ def triplet_delta_dis(self, ref, x1, x2): def fit_dis_embed( - inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu" + inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu", + noisy_method=None, parameter=None, robust_loss=None, itr=1, all_sols=None ): + # 这个函数使用 triplet-based contrastive learning，训练一个模型在嵌入空间中表示“人类感知下的多样性” + # inputs是随机产生的角度 + # print(device) + # print(inputs.shape) (250, 3, 10) + # print(gt_measures.shape) (250, 3, 2) + itr = (itr-1)/500 t = time.time() model = DisEmbed(input_dim=inputs.shape[-1], latent_dim=latent_dim) model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) loss_fn = lambda y, delta_dis: torch.max( torch.tensor([0.0]), 0.05 - y * delta_dis - ).mean() + ).mean() # hinge triplet loss n_pref_data = inputs.shape[0] ref = inputs[:, 0] x1 = inputs[:, 1] @@ -76,12 +86,16 @@ def fit_dis_embed( ref_gt_val = ref_gt_measures[n_train:] x1_gt_val = x1_gt_measures[n_train:] x2_gt_val = x2_gt_measures[n_train:] - # ref_gt_val = ref_gt_train - # x1_gt_val = x1_gt_train - # x2_gt_val = x2_gt_train - # best_acc = 0 - # counter = 0 + gt_dis_all = np.sum( + (np.square(ref_gt_train - x1_gt_train) - np.square(ref_gt_train - x2_gt_train)), + axis=-1 + ) + gt_all = torch.tensor(gt_dis_all > 0, dtype=torch.float32) * 2 - 1 + noise_gen = NoiseGenerator() + gt_noise_all = noise_gen.generate_noise( + gt_all, gt_dis_all, noisy_method=noisy_method, parameter=parameter + ) val_acc = [] for epoch in range(1000): for _ in range(n_iters_per_epoch): @@ -89,30 +103,176 @@ def fit_dis_embed( batch_ref = torch.tensor(ref_train[idx], dtype=torch.float32).to(device) batch1 = torch.tensor(x1_train[idx], dtype=torch.float32).to(device) batch2 = torch.tensor(x2_train[idx], dtype=torch.float32).to(device) - - optimizer.zero_grad() - delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) - gt_dis = np.sum( - ( - np.square(ref_gt_train[idx] - x1_gt_train[idx]) - - np.square(ref_gt_train[idx] - x2_gt_train[idx]) - ), - -1, - ) - gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 - - loss = loss_fn(gt, delta_dis) + batch_gt_noise = gt_noise_all[idx].to(device) # .clone().detach() + + # optimizer.zero_grad() + + # gt_dis = np.sum( + # ( + # np.square(ref_gt_train[idx] - x1_gt_train[idx]) + # - np.square(ref_gt_train[idx] - x2_gt_train[idx]) + # ), + # -1, + # ) + # print("abs(gt_dis) percentiles:",np.percentile(np.abs(gt_dis), [0, 5, 10, 20, 30, 50, 60, 70, 90, 100])) + # noise_gen = NoiseGenerator() + # gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 # 产生无噪声标签 + # gt_noise = noise_gen.generate_noise( + # gt, gt_dis, noisy_method=noisy_method, parameter=parameter) + + # loss = loss_fn(gt_noise, delta_dis) + + # 使用 Reweighted loss + if robust_loss == 'crdo': + optimizer.zero_grad() + ref_embed = model.forward(batch_ref).to(device) + x1_embed = model.forward(batch1).to(device) + x2_embed = model.forward(batch2).to(device) + triplet_loss_fn = TripletCDRP(gamma=5.0, eta=0.5, eps=0.01).to(device) + loss = triplet_loss_fn(ref_embed, x1_embed, x2_embed, batch_gt_noise).to(device) + if robust_loss == 'robust_qdhf': + # model.eval() + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2).detach() + signed_delta = delta_dis * batch_gt_noise + neg_mask = signed_delta < 0 + signed_delta_neg = signed_delta[neg_mask] + if signed_delta_neg.numel() > 0: + mean_neg = signed_delta_neg.mean() + std_neg = signed_delta_neg.std() + threshold1 = mean_neg - 3 * std_neg + threshold2 = mean_neg - 1 * std_neg + flip_mask = signed_delta <= threshold1 + replace_mask = (signed_delta > threshold1) & (signed_delta < threshold2) + keep_mask = signed_delta >= threshold2 + batch_gt_noise = batch_gt_noise.detach() + batch_gt_noise[flip_mask] = -batch_gt_noise[flip_mask] + for i in torch.where(replace_mask)[0]: + # with torch.no_grad(): + batch1[i], batch2[i], batch_gt_noise[i] = replace_sample( + batch_ref[i], batch1[i], batch2[i], + model=model, + device=device, + K=2, + noisy_method=noisy_method, + parameter=parameter, + all_sols=all_sols, + itr=itr + ) + # batch1[i] = b1.detach() + # batch2[i] = b2.detach() + # batch_gt_noise[i] = new_lbl.detach() + optimizer.zero_grad() + # model.train() + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + loss_agent = RobustLossAgent(margin=0.05) + loss = loss_agent.robust_loss(delta_dis, batch_gt_noise, robust_loss, parameter, epoch, device).to(device) + # loss.backward() + # optimizer.step() + else: + optimizer.zero_grad() + loss_agent = RobustLossAgent(margin=0.05) + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + loss = loss_agent.robust_loss(delta_dis, batch_gt_noise, robust_loss, parameter, epoch, device).to(device) loss.backward() optimizer.step() + # for epoch in range(1000): + # if epoch < 100: + # early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=5, delta=1e-1) + # else: + # early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=5, delta=3e-2) + # if robust_loss == 'ies': + # full_idx = np.arange(n_train) + # active_idx = early_stopper.get_active_indices(full_idx) + + # if epoch % 100 == 0: + # print(len(active_idx)) + + # if len(active_idx) <= batch_size: + # print("All triplets stopped early.") + # break + + # batch_idx = np.random.choice(active_idx, batch_size) + + # batch_ref = torch.tensor(ref_train[batch_idx], dtype=torch.float32).to(device) + # batch1 = torch.tensor(x1_train[batch_idx], dtype=torch.float32).to(device) + # batch2 = torch.tensor(x2_train[batch_idx], dtype=torch.float32).to(device) + + # ref_embed = model.forward(batch_ref) + # x1_embed = model.forward(batch1) + # x2_embed = model.forward(batch2) + + # gt_dis = np.sum( + # np.square(ref_gt_train[batch_idx] - x1_gt_train[batch_idx]) - + # np.square(ref_gt_train[batch_idx] - x2_gt_train[batch_idx]), + # axis=-1 + # ) + # gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 + # noise_gen = NoiseGenerator() + # gt_noise = noise_gen.generate_noise(gt, gt_dis, noisy_method=noisy_method, parameter=parameter).to(device) + + # delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + # loss_samplewise = torch.clamp(0.05 - gt_noise * delta_dis, min=0.0) # shape: (B,) + + # # if epoch <= 3: + # # print(f"Mean: {loss_samplewise.mean().item()}") + # # print(f"Standard Deviation: {loss_samplewise.std().item()}") + # # print(f"Max: {loss_samplewise.max().item()}") + # # print(f"Min: {loss_samplewise.min().item()}") + + # loss = loss_samplewise.mean() + + # optimizer.zero_grad() + # loss.backward() + # optimizer.step() + + # # 更新 IES 模块 + # early_stopper.update(batch_idx, loss_samplewise.detach().cpu()) + # else: + # for _ in range(n_iters_per_epoch): + # idx = np.random.choice(n_train, batch_size) + # batch_ref = torch.tensor(ref_train[idx], dtype=torch.float32).to(device) + # batch1 = torch.tensor(x1_train[idx], dtype=torch.float32).to(device) + # batch2 = torch.tensor(x2_train[idx], dtype=torch.float32).to(device) + + # optimizer.zero_grad() + + # gt_dis = np.sum( + # ( + # np.square(ref_gt_train[idx] - x1_gt_train[idx]) + # - np.square(ref_gt_train[idx] - x2_gt_train[idx]) + # ), + # -1, + # ) + # # print("abs(gt_dis) percentiles:",np.percentile(np.abs(gt_dis), [0, 5, 10, 20, 30, 50, 60, 70, 90, 100])) + # noise_gen = NoiseGenerator() + # gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 # 产生无噪声标签 + # gt_noise = noise_gen.generate_noise( + # gt, gt_dis, noisy_method=noisy_method, parameter=parameter) + + # # loss = loss_fn(gt_noise, delta_dis) + + # # 使用 Reweighted loss + # if robust_loss == 'crdo': + # ref_embed = model.forward(batch_ref).to(device) + # x1_embed = model.forward(batch1).to(device) + # x2_embed = model.forward(batch2).to(device) + # triplet_loss_fn = TripletCDRP(gamma=5.0, eta=0.5, eps=0.01).to(device) + # loss = triplet_loss_fn(ref_embed, x1_embed, x2_embed, gt_noise).to(device) + # else: + # loss_agent = RobustLossAgent(margin=0.05) + # delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + # loss = loss_agent.robust_loss(delta_dis, gt_noise, robust_loss, parameter, epoch, device).to(device) + # loss.backward() + # optimizer.step() # Evaluate. n_correct = 0 n_total = 0 with torch.no_grad(): idx = np.arange(n_val) - batch_ref = torch.tensor(ref_val[idx], dtype=torch.float32) - batch1 = torch.tensor(x1_val[idx], dtype=torch.float32) - batch2 = torch.tensor(x2_val[idx], dtype=torch.float32) + batch_ref = torch.tensor(ref_val[idx], dtype=torch.float32).to(device) + batch1 = torch.tensor(x1_val[idx], dtype=torch.float32).to(device) + batch2 = torch.tensor(x2_val[idx], dtype=torch.float32).to(device) delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) pred = delta_dis > 0 gt_dis = np.sum( @@ -122,7 +282,7 @@ def fit_dis_embed( ), -1, ) - gt = torch.tensor(gt_dis > 0) + gt = torch.tensor(gt_dis > 0).to(device) n_correct += (pred == gt).sum().item() n_total += len(idx) diff --git a/draw_pic.py b/draw_pic.py new file mode 100644 index 0000000..0b8fcac --- /dev/null +++ b/draw_pic.py @@ -0,0 +1,72 @@ +import os +import pandas as pd +import matplotlib.pyplot as plt +import argparse + +def plot_across_methods(folders, noise_type): + labels = [os.path.basename(folder.rstrip('/')).replace('_logs', '').capitalize() for folder in folders] + colors = ["blue", "green", "orange", "red", "purple", "cyan"] # 可扩展 , + fig_qd, ax_qd = plt.subplots() + fig_cov, ax_cov = plt.subplots() + + for folder, label, color in zip(folders, labels, colors): + filename = f"{noise_type}_experiment_results.csv" + file_path = os.path.join(folder, filename) + + if not os.path.exists(file_path): + print(f"文件 {file_path} 不存在，跳过。") + continue + + df = pd.read_csv(file_path) + + df_mean = df[df['Seed'] == 'Mean'].copy() + df_std = df[df['Seed'] == 'STD'].copy() + + df_mean.loc[:, 'Parameter'] = df_mean['Parameter'].astype(float) + df_std.loc[:, 'Parameter'] = df_std['Parameter'].astype(float) + + df_merged = pd.merge(df_mean, df_std, on='Parameter', suffixes=('_mean', '_std')) + df_merged = df_merged.sort_values(by='Parameter') + + x = df_merged['Parameter'] + qd_mean = df_merged['QD Score_mean'] + qd_std = df_merged['QD Score_std'] + cov_mean = df_merged['Coverage_mean'] + cov_std = df_merged['Coverage_std'] + + ax_qd.plot(x, qd_mean, label=label, color=color, marker='o', ms=3) + ax_qd.fill_between(x, qd_mean - qd_std, qd_mean + qd_std, color=color, alpha=0.2, linewidth=0) + + ax_cov.plot(x, cov_mean, label=label, color=color, marker='o', ms=3) + ax_cov.fill_between(x, cov_mean - cov_std, cov_mean + cov_std, color=color, alpha=0.2, linewidth=0) + + ax_qd.set_title(f"Impact of Noise Rate on QD Score ({noise_type.replace('_', ' ').title()})") + ax_qd.set_xlabel("Noise Rate") + ax_qd.set_ylabel("QD Score") + ax_qd.legend() + ax_qd.grid(True) + + ax_cov.set_title(f"Impact of Noise Rate on Coverage ({noise_type.replace('_', ' ').title()})") + ax_cov.set_xlabel("Noise Rate") + ax_cov.set_ylabel("Coverage") + ax_cov.legend() + ax_cov.grid(True) + + # 保存 + qd_path = f"arm/pic/qd_score_{noise_type}.png" + cov_path = f"arm/pic/coverage_{noise_type}.png" + fig_qd.savefig(qd_path, dpi=300, bbox_inches='tight') + fig_cov.savefig(cov_path, dpi=300, bbox_inches='tight') + print(f"图像已保存为：\n- {qd_path}\n- {cov_path}") + +# argparse 修改 +parser = argparse.ArgumentParser(description="Compare robust methods under a specific noise type") +parser.add_argument('--noise_type', type=str, required=True, help="e.g. flip_labels_asymmetric") +parser.add_argument('--log_dirs', nargs='+', required=True, help="List of robust method folders") + +args = parser.parse_args() +plot_across_methods(args.log_dirs, args.noise_type) + +# python draw_pic.py --noise_type noisy_labels_exact --log_dirs /mnt/data6t/qdhf/arm/logs/reweight_logs /mnt/data6t/qdhf/arm/logs/rDPO_logs /mnt/data6t/qdhf/arm/logs/label_smoothing_logs /mnt/data6t/qdhf/arm/logs/raw_qdhf_logs /mnt/data6t/qdhf/arm/logs/cDPO_logs /mnt/data6t/qdhf/arm/logs/crdo_logs +# python draw_pic.py --noise_type flip_labels_asymmetric --log_dirs /mnt/data6t/qdhf/arm/logs/reweight_logs /mnt/data6t/qdhf/arm/logs/rDPO_logs /mnt/data6t/qdhf/arm/logs/label_smoothing_logs /mnt/data6t/qdhf/arm/logs/raw_qdhf_logs /mnt/data6t/qdhf/arm/logs/cDPO_logs /mnt/data6t/qdhf/arm/logs/crdo_logs +# python draw_pic.py --noise_type stochastic --log_dirs /mnt/data6t/qdhf/arm/logs/reweight_logs /mnt/data6t/qdhf/arm/logs/rDPO_logs /mnt/data6t/qdhf/arm/logs/label_smoothing_logs /mnt/data6t/qdhf/arm/logs/raw_qdhf_logs /mnt/data6t/qdhf/arm/logs/cDPO_logs /mnt/data6t/qdhf/arm/logs/crdo_logs \ No newline at end of file diff --git a/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/0f48b2256e823029112effea0c3586bf369da56c.lock b/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/0f48b2256e823029112effea0c3586bf369da56c.lock new file mode 100644 index 0000000..e69de29 diff --git a/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/5294955ff7801083f720b34b55d0f1f51313c5c5.lock b/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/5294955ff7801083f720b34b55d0f1f51313c5c5.lock new file mode 100644 index 0000000..e69de29 diff --git a/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/b68a817036d3f5c36537e9c332a42c7e63a9d4e9.lock b/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/b68a817036d3f5c36537e9c332a42c7e63a9d4e9.lock new file mode 100644 index 0000000..e69de29 diff --git a/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/cce6febb0b6d876ee5eb24af35e27e764eb4f9b1d0b7c026c8c3333d4cfc916c.lock b/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/cce6febb0b6d876ee5eb24af35e27e764eb4f9b1d0b7c026c8c3333d4cfc916c.lock new file mode 100644 index 0000000..e69de29 diff --git a/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/f9ac7c0b3b341525cd1b65e1a715128fee97b25c.lock b/lsi/.locks/models--stabilityai--stable-diffusion-2-1-base/f9ac7c0b3b341525cd1b65e1a715128fee97b25c.lock new file mode 100644 index 0000000..e69de29 diff --git a/lsi/check_model.py b/lsi/check_model.py new file mode 100644 index 0000000..6b21bd4 --- /dev/null +++ b/lsi/check_model.py @@ -0,0 +1,21 @@ +import os + +# 禁用 hf_transfer 下载加速 +os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "false" + +from diffusers import StableDiffusionPipeline +import torch + +model_id = "stabilityai/stable-diffusion-2-1-base" +torch_dtype = torch.float16 # 设置你需要的torch类型 + +# 加载模型 +pipe = StableDiffusionPipeline.from_pretrained( + model_id, + torch_dtype=torch_dtype, + safety_checker=None, + requires_safety_checker=False +) + +# 打印模型路径确认 +print(pipe._model.config._name_or_path) diff --git a/lsi/collage.py b/lsi/collage.py old mode 100644 new mode 100755 index 35dd9a1..7171e3b --- a/lsi/collage.py +++ b/lsi/collage.py @@ -30,7 +30,7 @@ def make_archive_collage(archive_filename, prompt, save_all=False): gen_output_dir = os.path.join("logs/vis", archive_filename.split("/")[1]) gen_output_dir = Path(gen_output_dir) if not gen_output_dir.is_dir(): - gen_output_dir.mkdir() + gen_output_dir.mkdir(parents=True, exist_ok=True) model_id = "stabilityai/stable-diffusion-2-1-base" diff --git a/lsi/generate_noise.py b/lsi/generate_noise.py new file mode 100755 index 0000000..77b87aa --- /dev/null +++ b/lsi/generate_noise.py @@ -0,0 +1,77 @@ +import torch + +class NoiseGenerator: + def __init__(self): + pass + + def generate_noise(self, gt, gt_dis, noisy_method, parameter): + if noisy_method == 'stochastic': + return self.stochastic_labels(gt, gt_dis, parameter) + elif noisy_method == 'noisy_labels_exact': + return self.noisy_labels_exact(gt, parameter) + elif noisy_method == 'add_equal_noise': + return self.add_equal_noise(gt, gt_dis, parameter) + elif noisy_method == 'flip_by_distance': + return self.flip_by_distance(gt, gt_dis, parameter) + elif noisy_method == 'flip_labels_asymmetric': + return self.flip_labels_asymmetric(gt, parameter) + else: + raise ValueError(f"Unknown noisy_method: {noisy_method}") + + def noisy_labels_exact(self, gt, noise_rate): + n = len(gt) + n_flip = int(n * noise_rate) + flip_indices = torch.randperm(n)[:n_flip] + gt_noisy = gt.clone() + gt_noisy[flip_indices] = -gt_noisy[flip_indices] + return gt_noisy + + def stochastic_labels(self, gt, gt_dis, noise_ratio): + gt = gt.clone() + n = len(gt) + n_noise = int(n * noise_ratio) + + noise_indices = torch.randperm(n)[:n_noise] + + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + prob = torch.sigmoid(-gt_dis_tensor[noise_indices]) + sampled = torch.bernoulli(prob) + noisy_labels = sampled * 2 - 1 + + gt[noise_indices] = noisy_labels + return gt + + def add_equal_noise(self, gt, gt_dis, delta_equal): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + equal_mask = torch.abs(gt_dis_tensor) < delta_equal + + gt_noisy = gt.clone() + gt_noisy[equal_mask] = 0 + return gt_noisy + + def flip_by_distance(self, gt, gt_dis, delta_threshold): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + flip_mask = torch.abs(gt_dis_tensor) < delta_threshold + + gt_noisy = gt.clone() + gt_noisy[flip_mask] = -gt_noisy[flip_mask] + return gt_noisy + + def flip_labels_asymmetric(self, gt, flip_rate=0.1): + flip_rate_neg = flip_rate + flip_rate_pos = flip_rate + gt = gt.clone() + pos_indices = torch.where(gt == 1)[0] + neg_indices = torch.where(gt == -1)[0] + + n_pos_flip = int(len(pos_indices) * flip_rate_pos) + n_neg_flip = int(len(neg_indices) * flip_rate_neg) + + pos_flip_idx = pos_indices[torch.randperm(len(pos_indices))[ + :n_pos_flip]] + neg_flip_idx = neg_indices[torch.randperm(len(neg_indices))[ + :n_neg_flip]] + + gt[pos_flip_idx] = -1 + gt[neg_flip_idx] = 1 + return gt diff --git a/lsi/lsi_utils.py b/lsi/lsi_utils.py old mode 100644 new mode 100755 index 1c2335f..b2792c1 --- a/lsi/lsi_utils.py +++ b/lsi/lsi_utils.py @@ -4,7 +4,8 @@ import torch from sklearn.decomposition import PCA from torch import nn - +from generate_noise import NoiseGenerator +from robust_loss import RobustLossAgent, TripletCDRP, InstanceEarlyStopper class DisEmbed(nn.Module): def __init__(self, input_dim, latent_dim): @@ -32,7 +33,7 @@ def triplet_delta_dis(self, ref, x1, x2): def fit_dis_embed( - inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu" + inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu", noisy_method=None, parameter=None, robust_loss=None ): t = time.time() model = DisEmbed(input_dim=inputs.shape[-1], latent_dim=latent_dim) @@ -70,25 +71,94 @@ def fit_dis_embed( val_acc = [] for epoch in range(1000): - for _ in range(n_iters_per_epoch): - idx = np.random.choice(n_train, batch_size) - batch_ref = ref_train[idx].float() - batch1 = x1_train[idx].float() - batch2 = x2_train[idx].float() + if epoch < 100: + early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=3, delta=1e-1) + else: + early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=3, delta=5e-2) + if robust_loss == 'ies': + full_idx = np.arange(n_train) + active_idx = early_stopper.get_active_indices(full_idx) + + if len(active_idx) <= batch_size: + print("All triplets stopped early.") + break + + batch_idx = np.random.choice(active_idx, batch_size) + + batch_ref = torch.tensor(ref_train[batch_idx], dtype=torch.float32).to(device) + batch1 = torch.tensor(x1_train[batch_idx], dtype=torch.float32).to(device) + batch2 = torch.tensor(x2_train[batch_idx], dtype=torch.float32).to(device) + + ref_embed = model.forward(batch_ref) + x1_embed = model.forward(batch1) + x2_embed = model.forward(batch2) + + gt_dis = np.sum( + np.square(ref_gt_train[batch_idx].cpu().numpy() - x1_gt_train[batch_idx].cpu().numpy()) - + np.square(ref_gt_train[batch_idx].cpu().numpy() - x2_gt_train[batch_idx].cpu().numpy()), + axis=-1 + ) + gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 + noise_gen = NoiseGenerator() + gt_noise = noise_gen.generate_noise(gt, gt_dis, noisy_method=noisy_method, parameter=parameter).to(device) - optimizer.zero_grad() delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) - gt_dis = torch.nn.functional.cosine_similarity( - ref_gt_train[idx], x2_gt_train[idx], dim=-1 - ) - torch.nn.functional.cosine_similarity( - ref_gt_train[idx], x1_gt_train[idx], dim=-1 - ) - gt = (gt_dis > 0).float() * 2 - 1 + loss_samplewise = torch.clamp(0.05 - gt_noise * delta_dis, min=0.0) # shape: (B,) + + # if epoch <= 3: + # print(f"Mean: {loss_samplewise.mean().item()}") + # print(f"Standard Deviation: {loss_samplewise.std().item()}") + # print(f"Max: {loss_samplewise.max().item()}") + # print(f"Min: {loss_samplewise.min().item()}") + + loss = loss_samplewise.mean() - loss = loss_fn(gt, delta_dis) + optimizer.zero_grad() loss.backward() optimizer.step() + # 更新 IES 模块 + early_stopper.update(batch_idx, loss_samplewise.detach().cpu()) + else: + for _ in range(n_iters_per_epoch): + idx = np.random.choice(n_train, batch_size) + batch_ref = torch.tensor(ref_train[idx], dtype=torch.float32).to(device) + batch1 = torch.tensor(x1_train[idx], dtype=torch.float32).to(device) + batch2 = torch.tensor(x2_train[idx], dtype=torch.float32).to(device) + + optimizer.zero_grad() + + gt_dis = np.sum( + ( + np.square(ref_gt_train[idx].cpu().numpy() - x1_gt_train[idx].cpu().numpy()) + - np.square(ref_gt_train[idx].cpu().numpy() - x2_gt_train[idx].cpu().numpy()) + ), + -1, + ) + # print("abs(gt_dis) percentiles:",np.percentile(np.abs(gt_dis), [0, 5, 10, 20, 30, 50, 60, 70, 90, 100])) + noise_gen = NoiseGenerator() + gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 # 产生无噪声标签 + gt_noise = noise_gen.generate_noise( + gt, gt_dis, noisy_method=noisy_method, parameter=parameter) + + # loss = loss_fn(gt_noise, delta_dis) + + # 使用 Reweighted loss + if robust_loss == 'crdo': + ref_embed = model.forward(batch_ref).to(device) + x1_embed = model.forward(batch1).to(device) + x2_embed = model.forward(batch2).to(device) + triplet_loss_fn = TripletCDRP(gamma=5.0, eta=0.5, eps=0.01).to(device) + loss = triplet_loss_fn(ref_embed, x1_embed, x2_embed, gt_noise).to(device) + else: + loss_agent = RobustLossAgent(margin=0.05) + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + delta_dis = delta_dis.to(device) + gt_noise = gt_noise.to(device) + loss = loss_agent.robust_loss(delta_dis, gt_noise, robust_loss, parameter, epoch, device) + loss.backward() + optimizer.step() + # Evaluate. n_correct = 0 n_total = 0 diff --git a/lsi/main.py b/lsi/main.py old mode 100644 new mode 100755 index 6ebd939..833525b --- a/lsi/main.py +++ b/lsi/main.py @@ -6,7 +6,12 @@ import sys import time from pathlib import Path +import subprocess +import pandas as pd +import numpy as np +import re +import argparse import clip import matplotlib import numpy as np @@ -57,7 +62,7 @@ def evaluate_lsi( device="cpu", return_features=False, clip_features=None, - dreamsim_features=None, + dreamsim_features=None, # human preference model objs=None, ): # print(latents.shape) # torch.Size([10, 4, 64, 64]) @@ -258,15 +263,21 @@ def run_experiment( n_pref_data=100, online_finetune=False, incre_bounds=False, + noisy_method=None, + parameter=None, + robust_loss=None, + device='cpu' ): + os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "false" algorithm = "map_elites" - device = "cuda" if torch.cuda.is_available() else "cpu" + # device = "cuda" if torch.cuda.is_available() else "cpu" np.random.seed(seed) - torch.manual_seed(seed) + torch.manual_seed(seed) # + parameter_str = str(parameter).replace(".", "_") # Create a directory for this specific trial. - s_logdir = os.path.join(outdir, f"{algorithm}_{prompt}") + s_logdir = os.path.join(outdir, f"{noisy_method}_{parameter_str}_{seed}") logdir = Path(s_logdir) if not logdir.is_dir(): logdir.mkdir() @@ -394,6 +405,9 @@ def run_experiment( latent_dim=2, seed=seed, device=device, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss ) else: dis_embed = None @@ -446,6 +460,9 @@ def run_experiment( latent_dim=2, seed=seed, device=device, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss ) metadata["dis_embed"] = dis_embed @@ -526,28 +543,113 @@ def run_experiment( print(log_file_name, "| QD score:", data[1], "Coverage:", data[2]) print() + + csv_path = f"logs/{method}_{noisy_method}_experiment_results.csv" + header_written = not os.path.exists(csv_path) + record = { + "Method": method, + "Parameter": parameter, + "Seed": seed, + "QD Score": qd_score, + "Coverage": coverage + } + pd.DataFrame([record]).to_csv(csv_path, mode='a', header=header_written, index=False) + + if seed == 4444: + df = pd.read_csv(csv_path) + last_four_rows = df.tail(4) + + qd_scores = last_four_rows['QD Score'].values + coverage_scores = last_four_rows['Coverage'].values + + qd_mean = np.nanmean(qd_scores) + qd_std = np.nanstd(qd_scores) + coverage_mean = np.nanmean(coverage_scores) + coverage_std = np.nanstd(coverage_scores) + stats_record = { + "Method": method, + "Parameter": parameter, + "Seed": "Mean", + "QD Score": qd_mean, + "Coverage": coverage_mean + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + stats_record = { + "Method": method, + "Parameter": parameter, + "Seed": "STD", + "QD Score": qd_std, + "Coverage": coverage_std + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + print(f"Experiment results for seed {seed} have been saved to {csv_path}") + if __name__ == "__main__": - if len(sys.argv) > 1: - prompt = sys.argv[1] + parser = argparse.ArgumentParser( + description="Run robot arm experiments") + # parser.add_argument('--trial_id', type=int, default=0) + parser.add_argument('--prompt', type=str, default='a photo of an astronaut riding a horse on mars') + parser.add_argument('--noisy_method', type=str, choices=['stochastic', 'add_equal_noise', + 'flip_by_distance', 'flip_labels_asymmetric', 'noisy_labels_exact'], required=True) + parser.add_argument('--parameter', type=float, required=True) + parser.add_argument('--robust_loss',type=str,required=True) + parser.add_argument('--seed', type=int, required=False) + parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') + parser.add_argument('--cuda_index', type=int, default=0) + + args = parser.parse_args() + + if args.device == 'cuda': + if torch.cuda.is_available(): + device = torch.device(f'cuda:{args.cuda_index}') + else: + print("⚠️ CUDA was requested but is not available. Falling back to CPU.") + device = torch.device('cpu') else: - prompt = "a photo of an astronaut riding a horse on mars" + device = torch.device('cpu') # Create a shared logging directory for the experiments for this algorithm. - outdir = Path("logs") - if not outdir.is_dir(): - outdir.mkdir() - - run_experiment( - "qdhf", - prompt, - use_dis_embed=True, - n_pref_data=10000 // 4, - online_finetune=True, - ) + + out_dir = f'logs/{args.robust_loss}_logs' + os.makedirs(out_dir, exist_ok=True) + + seeds = [1111, 2222, 3333, 4444] + noisy_list = { + "noisy_labels_exact": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "stochastic": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "add_equal_noise": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_by_distance": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_labels_asymmetric": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +} + + + noisy_method = args.noisy_method + noisy_rates = noisy_list[noisy_method] + + for noisy_rate in noisy_rates: + for seed in seeds: + print(f"Running experiment with seed {seed}") + run_experiment( + "qdhf", + args.prompt, + outdir=out_dir, + itrs=1000, + use_dis_embed=True, + n_pref_data=10000 // 4, + online_finetune=True, + noisy_method=noisy_method, + parameter=noisy_rate, + seed=seed, + robust_loss=args.robust_loss, + device=device + ) - archive_filename = ( - f"logs/map_elites_{prompt}/qdhf(n=10000)|online|fixed_archive_00000200.pkl" - ) - make_archive_collage(archive_filename, prompt) + parameter_str = str(args.parameter).replace(".", "_") + archive_filename = ( + f"logs/map_elites_{args.prompt}/qdhf(n=10000)|online|fixed_archive_00000200.pkl" + ) + # make_archive_collage(archive_filename, args.prompt) diff --git a/lsi/ribs/__init__.py b/lsi/ribs/__init__.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/__init__.py b/lsi/ribs/archives/__init__.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/_add_status.py b/lsi/ribs/archives/_add_status.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/_archive_base.py b/lsi/ribs/archives/_archive_base.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/_cvt_archive.py b/lsi/ribs/archives/_cvt_archive.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/_grid_archive.py b/lsi/ribs/archives/_grid_archive.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/archives/_sliding_boundaries_archive.py b/lsi/ribs/archives/_sliding_boundaries_archive.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/__init__.py b/lsi/ribs/emitters/__init__.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_emitter_base.py b/lsi/ribs/emitters/_emitter_base.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_gaussian_emitter.py b/lsi/ribs/emitters/_gaussian_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_gradient_emitter.py b/lsi/ribs/emitters/_gradient_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_gradient_improvement_emitter.py b/lsi/ribs/emitters/_gradient_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_improvement_emitter.py b/lsi/ribs/emitters/_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_iso_line_emitter.py b/lsi/ribs/emitters/_iso_line_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_optimizing_emitter.py b/lsi/ribs/emitters/_optimizing_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/_random_direction_emitter.py b/lsi/ribs/emitters/_random_direction_emitter.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/opt/__init__.py b/lsi/ribs/emitters/opt/__init__.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/opt/_adam.py b/lsi/ribs/emitters/opt/_adam.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/emitters/opt/_cma_es.py b/lsi/ribs/emitters/opt/_cma_es.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/factory.py b/lsi/ribs/factory.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/optimizers/__init__.py b/lsi/ribs/optimizers/__init__.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/optimizers/_optimizer.py b/lsi/ribs/optimizers/_optimizer.py old mode 100644 new mode 100755 diff --git a/lsi/ribs/visualize.py b/lsi/ribs/visualize.py old mode 100644 new mode 100755 diff --git a/lsi/robust_loss.py b/lsi/robust_loss.py new file mode 100644 index 0000000..ec09ff8 --- /dev/null +++ b/lsi/robust_loss.py @@ -0,0 +1,163 @@ +import torch +import torch.nn as nn + +class RobustLossAgent: + def __init__(self, margin=0.05): + self.margin = margin + + def robust_loss(self, delta_dis, y, robust_loss, parameter, robust_parameter=None, epoch=0): + robust_parameter = robust_parameter or {} + + if robust_loss == 'reweight': + beta = robust_parameter.get("beta", 0.2) + return self.reweighted_triplet_loss(delta_dis, y, beta=beta) + + elif robust_loss == 'truncated': + alpha = robust_parameter.get("alpha", 0.05) + epsilon_max = robust_parameter.get("epsilon_max", 0.2) + return self.truncated_triplet_loss(delta_dis, y, epoch, alpha=alpha, epsilon_max=epsilon_max) + + elif robust_loss == 'label_smoothing': + alpha = robust_parameter.get("alpha", 0.05) + return self.label_smoothing_triplet_loss(delta_dis, y, alpha=alpha) + + elif robust_loss == 'rDPO': + epsilon = parameter + return self.rDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'cDPO': + epsilon = parameter + return self.cDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'None': + loss_fn = lambda y, delta_dis: torch.max(torch.tensor([0.0], device=y.device), 0.05 - y * delta_dis).mean() + return loss_fn(y, delta_dis) + + else: + raise ValueError(f"unknown robust method: {robust_loss}") + + def reweighted_triplet_loss(self, delta_dis, y, beta=0.2, eps=1e-6): + pred = torch.sigmoid(delta_dis) + weight = pred ** beta + raw_loss = weight * torch.clamp(self.margin - y * delta_dis, min=0.0) + normalized_loss = raw_loss.sum() * (delta_dis.size(0) / (weight.sum() + eps)) + return normalized_loss + + def truncated_triplet_loss(self, delta_dis, y, epoch, alpha=0.1, epsilon_max=0.1): + loss_sample = torch.clamp(self.margin - y * delta_dis, min=0.0) + drop_rate = min(alpha * epoch, epsilon_max) + batch_size = loss_sample.size(0) + + if drop_rate > 0: + kth = int(batch_size * (1 - drop_rate)) + if kth <= 0: + loss_sample = torch.zeros_like(loss_sample) + else: + tau = torch.kthvalue(loss_sample, kth).values + keep_mask = (loss_sample <= tau) + loss_sample = loss_sample * keep_mask.float() + + return loss_sample.mean() + + + def smooth_labels(self, y, alpha=0.1, num_classes=2): + if alpha <= 1e-6: + return y, y # 不平滑 + + y_idx = ((y + 1) / 2).long() + I = torch.eye(num_classes, device=y.device, dtype=y.dtype) + J = torch.ones((num_classes, num_classes), device=y.device, dtype=y.dtype) + M = (1 - alpha) * I + (alpha / num_classes) * J + + soft_labels = M[y_idx] + y_smooth = soft_labels[:, 1] - soft_labels[:, 0] # 转换为 ∈ [-1, 1] + y_smooth = torch.clamp(y_smooth, -1.0, 1.0) + return soft_labels, y_smooth + + + def label_smoothing_triplet_loss(self, delta_dis, y, alpha): + _, y_smooth = self.smooth_labels(y, alpha=alpha, num_classes=2) + loss = torch.clamp(self.margin - y_smooth * delta_dis, min=0.0) + return loss.mean() + + + def rDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + # print(epsilon) + y = y.to(delta_dis.device) + loss_clean = -torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + loss_flipped = -torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + numerator = (1 - epsilon) * loss_clean - epsilon * loss_flipped + denominator = 1 - 2 * epsilon + corrected_loss = numerator / (denominator + 1e-8) # add epsilon to avoid div 0 + return corrected_loss.mean() + + def cDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + y = y.to(delta_dis.device) + log_prob_clean = torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + log_prob_flipped = torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + # Weighted sum per noisy label distribution + loss = -((1 - epsilon) * log_prob_clean + epsilon * log_prob_flipped) + return loss.mean() + + + +class TripletCDRP(nn.Module): + def __init__(self, gamma=5.0, eta=0.5, eps=0.01): + super(TripletCDRP, self).__init__() + self.gamma = gamma + self.eta = eta + self.eps = eps + + def forward(self, ref_embed, x1_embed, x2_embed, pref_label): + d1 = torch.sum((ref_embed - x1_embed) ** 2, dim=-1) + d2 = torch.sum((ref_embed - x2_embed) ** 2, dim=-1) + logits = torch.stack([-d1, -d2], dim=1) # (B, 2) + logits = torch.clamp(logits, min=-20, max=20) + probs = torch.softmax(logits, dim=1) # (B, 2) + + labels = ((pref_label + 1) // 2).long() + + # Base CE loss + ce_loss = nn.NLLLoss(reduction="none") + log_probs = torch.log(probs + 1e-8) + # nominal_loss = ce_loss(log_probs, labels) # (B,) + + # C(y', y) = gamma if y' != y else 0 + gamma = self.gamma + penalty = gamma * (1 - torch.nn.functional.one_hot(labels, num_classes=2).to(logits.device)) + + # robust loss inner: max_y' [l(y') - gamma * 1_{y' ≠ y}] + loss_all = ce_loss(log_probs.repeat(1, 2).reshape(-1, 2), torch.tensor([0, 1] * logits.shape[0]).to(logits.device)).reshape(-1, 2) + robust_loss = (loss_all - penalty).max(dim=1).values + + final_loss = self.eps * gamma + robust_loss.mean() + + return final_loss + + +class InstanceEarlyStopper: + def __init__(self, num_samples, patience=3, delta=1e-4): + self.patience = patience + self.delta = delta + self.loss_history = [[] for _ in range(num_samples)] + self.mask = torch.zeros(num_samples, dtype=torch.bool) # True 表示“停止训练该样本” + + def update(self, indices, losses): + for i, idx in enumerate(indices): + idx = int(idx) + if self.mask[idx]: + continue + self.loss_history[idx].append(losses[i].item()) + + if len(self.loss_history[idx]) >= self.patience: + l_hist = self.loss_history[idx][-self.patience:] + if len(l_hist) == self.patience: + delta2 = sum([-l_hist[j] + 2*l_hist[j+1] - l_hist[j+2] for j in range(self.patience - 2)]) + + if abs(delta2) < self.delta: + self.mask[idx] = True + + def get_active_indices(self, all_indices): + return [idx for idx in all_indices if not self.mask[int(idx)]] diff --git a/lsi/run_experiment.py b/lsi/run_experiment.py new file mode 100755 index 0000000..fb3c55f --- /dev/null +++ b/lsi/run_experiment.py @@ -0,0 +1,16 @@ +import subprocess +import os + +# 定义 noisy_labels_exact 和 seeds +noisy_labels_exact = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +seeds = ["1111", "2222", "3333", "4444"] + +# 遍历每个 noisy_labels_exact 的参数和每个 seed +for parameter in noisy_labels_exact: + for seed in seeds: + # 构建命令 + command = f"python main.py --noisy_method noisy_labels_exact --parameter {parameter} --seed {seed}" + + # 打印并执行命令 + print(f"Running: {command}") + subprocess.run(command, shell=True) diff --git a/lsi/run_lsi.sh b/lsi/run_lsi.sh new file mode 100755 index 0000000..acd47c3 --- /dev/null +++ b/lsi/run_lsi.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# export CUDA_VISIBLE_DEVICES=5 +#!/usr/bin/env bash +# File: run_tmux.sh +# Usage: bash run_tmux.sh + +SESSION="qd_hf_run" + +# List the five noise methods you want to try +METHODS=( + noisy_labels_exact + stochastic + #add_equal_noise + #flip_by_distance + flip_labels_asymmetric +) + +# ---- create / configure tmux session ---- +tmux kill-session -t "$SESSION" 2>/dev/null # 先删旧会话 +tmux new-session -d -s "$SESSION" # 再新建 +k=1 +for i in "${!METHODS[@]}"; do + + METHOD="${METHODS[$i]}" + WIN_NAME="$METHOD" + + if [[ $i -eq 0 ]]; then # window 0 already exists + tmux rename-window -t "$SESSION:0" "$WIN_NAME" + else + tmux new-window -t "$SESSION" -n "$WIN_NAME" + fi + + # build the command string for this window + CMD="source ~/.bashrc && conda activate QDHF && \ +python main.py --noisy_method ${METHOD} --seed 2222 --parameter 0.3 \ + --robust_loss None --device cuda --cuda_index $(((k%3)+1))" + + # send the command and press + tmux send-keys -t "$SESSION:$WIN_NAME" "$CMD" C-m + k=$((k+1)) + +done + +# Finally, attach to the session so you can watch the runs +tmux attach -t "$SESSION" + +#python run_experiment.py \ No newline at end of file diff --git a/lsi/version_diffusers_cache.txt b/lsi/version_diffusers_cache.txt new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/lsi/version_diffusers_cache.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/maze/generate_noise.py b/maze/generate_noise.py new file mode 100755 index 0000000..c0af89b --- /dev/null +++ b/maze/generate_noise.py @@ -0,0 +1,78 @@ +import torch + + +class NoiseGenerator: + def __init__(self): + pass + + def generate_noise(self, gt, gt_dis, noisy_method, parameter): + if noisy_method == 'stochastic': + return self.stochastic_labels(gt, gt_dis, parameter) + elif noisy_method == 'noisy_labels_exact': + return self.noisy_labels_exact(gt, parameter) + elif noisy_method == 'add_equal_noise': + return self.add_equal_noise(gt, gt_dis, parameter) + elif noisy_method == 'flip_by_distance': + return self.flip_by_distance(gt, gt_dis, parameter) + elif noisy_method == 'flip_labels_asymmetric': + return self.flip_labels_asymmetric(gt, parameter) + else: + raise ValueError(f"Unknown noisy_method: {noisy_method}") + + def noisy_labels_exact(self, gt, noise_rate): + n = len(gt) + n_flip = int(n * noise_rate) + flip_indices = torch.randperm(n)[:n_flip] + gt_noisy = gt.clone() + gt_noisy[flip_indices] = -gt_noisy[flip_indices] + return gt_noisy + + def stochastic_labels(self, gt, gt_dis, noise_ratio): + gt = gt.clone() + n = len(gt) + n_noise = int(n * noise_ratio) + + noise_indices = torch.randperm(n)[:n_noise] + + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + prob = torch.sigmoid(-gt_dis_tensor[noise_indices]) + sampled = torch.bernoulli(prob) + noisy_labels = sampled * 2 - 1 + + gt[noise_indices] = noisy_labels + return gt + + def add_equal_noise(self, gt, gt_dis, delta_equal): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + equal_mask = torch.abs(gt_dis_tensor) < delta_equal + + gt_noisy = gt.clone() + gt_noisy[equal_mask] = 0 + return gt_noisy + + def flip_by_distance(self, gt, gt_dis, delta_threshold): + gt_dis_tensor = torch.tensor(gt_dis, dtype=torch.float32) + flip_mask = torch.abs(gt_dis_tensor) < delta_threshold + + gt_noisy = gt.clone() + gt_noisy[flip_mask] = -gt_noisy[flip_mask] + return gt_noisy + + def flip_labels_asymmetric(self, gt, flip_rate=0.1): + flip_rate_neg = flip_rate + flip_rate_pos = flip_rate + gt = gt.clone() + pos_indices = torch.where(gt == 1)[0] + neg_indices = torch.where(gt == -1)[0] + + n_pos_flip = int(len(pos_indices) * flip_rate_pos) + n_neg_flip = int(len(neg_indices) * flip_rate_neg) + + pos_flip_idx = pos_indices[torch.randperm(len(pos_indices))[ + :n_pos_flip]] + neg_flip_idx = neg_indices[torch.randperm(len(neg_indices))[ + :n_neg_flip]] + + gt[pos_flip_idx] = -1 + gt[neg_flip_idx] = 1 + return gt diff --git a/maze/kheperax/__init__.py b/maze/kheperax/__init__.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/geoms.py b/maze/kheperax/geoms.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/laser.py b/maze/kheperax/laser.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/maze.py b/maze/kheperax/maze.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/posture.py b/maze/kheperax/posture.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/rendering_tools.py b/maze/kheperax/rendering_tools.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/robot.py b/maze/kheperax/robot.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/task.py b/maze/kheperax/task.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/tree_utils.py b/maze/kheperax/tree_utils.py old mode 100644 new mode 100755 diff --git a/maze/kheperax/type_fixer_wrapper.py b/maze/kheperax/type_fixer_wrapper.py old mode 100644 new mode 100755 diff --git a/maze/kill_dead.py b/maze/kill_dead.py new file mode 100644 index 0000000..7ae2b83 --- /dev/null +++ b/maze/kill_dead.py @@ -0,0 +1,20 @@ +import psutil + +# 假设你已经知道某个子进程的PID +child_pid = 3654792 # 这里替换为你的子进程 PID + +# 获取子进程的对象 +child_process = psutil.Process(child_pid) + +# 获取父进程的PID +parent_pid = child_process.ppid() + +# 获取父进程的所有子进程 +parent_process = psutil.Process(parent_pid) +for child in parent_process.children(recursive=True): + print(f"Terminating child process {child.pid}") + child.terminate() # 或者使用 child.kill() 强制终止 + +# 如果需要，终止父进程本身 +print(f"Terminating parent process {parent_pid}") +parent_process.terminate() # 或者使用 parent_process.kill() 强制终止 diff --git a/maze/main.py b/maze/main.py old mode 100644 new mode 100755 index 40d736a..53df03f --- a/maze/main.py +++ b/maze/main.py @@ -13,6 +13,7 @@ import matplotlib import numpy as np import torch +import argparse from alive_progress import alive_bar from kheperax.task import KheperaxConfig, KheperaxTask from qdax.core.emitters.mutation_operators import isoline_variation @@ -113,7 +114,7 @@ def create_optimizer( if method == "qd": assert gt_bounds is not None objs, measures = evaluate_maze( - sols, method, scoring_fn, metadata, random_key=random_key + sols, method, scoring_fn, metadata, device, random_key=random_key ) if archive_bounds is None: archive_bounds = gt_bounds @@ -223,9 +224,13 @@ def run_experiment( n_pref_data=1000, online_finetune=False, incre_bounds=False, + noisy_method=None, + parameter=None, + robust_loss=None, + device='cpu' ): algorithm = "map_elites" - device = "cpu" + # device = "cpu" batch_size = 200 num_evaluations = int(2e5) @@ -266,9 +271,9 @@ def run_experiment( iso_sigma=iso_sigma, line_sigma=line_sigma, ) - + num_parameter = str(parameter * 100) # Create a directory for this specific trial. - s_logdir = os.path.join(outdir, f"{algorithm}_trial_{trial_id}") + s_logdir = os.path.join(outdir, f"{noisy_method}_trial{trial_id}_{num_parameter}_{seed}") logdir = Path(s_logdir) if not logdir.is_dir(): logdir.mkdir() @@ -346,6 +351,7 @@ def run_experiment( _, gt_measures, features = evaluate_maze( inputs, method="qd", + device=device, scoring_fn=scoring_fn, return_features=True, random_key=random_key, @@ -358,6 +364,10 @@ def run_experiment( dis_embed_gt_measures, latent_dim=2, seed=seed, + noisy_method = noisy_method, + parameter = parameter, + robust_loss=robust_loss, + device=device ) else: dis_embed = None @@ -382,6 +392,7 @@ def run_experiment( scoring_fn=scoring_fn, return_features=True, random_key=random_key, + device=device ) additional_features = additional_features.reshape( n_pref_data, 3, -1 @@ -400,6 +411,10 @@ def run_experiment( dis_embed_gt_measures, latent_dim=2, seed=seed, + noisy_method = noisy_method, + parameter = parameter, + robust_loss=robust_loss, + device = device ) all_sols = list_to_batch(all_sols) @@ -441,6 +456,7 @@ def run_experiment( method="qd", scoring_fn=scoring_fn, random_key=random_key, + device=device ) for i in range(len(_objs)): gt_archive_all.add(1, _objs[i], _gt_measures[i]) @@ -462,6 +478,7 @@ def run_experiment( method="qd", scoring_fn=scoring_fn, random_key=random_key, + device=device ) for i in range(len(_objs)): gt_archive_all.add(1, _objs[i], _gt_measures[i]) @@ -530,6 +547,7 @@ def run_experiment( method="qd", scoring_fn=scoring_fn, random_key=random_key, + device=device ) for i in range(len(sols)): gt_archive.add(1, objs[i], gt_measures[i]) @@ -615,6 +633,11 @@ def arm_main( n_pref_data=1000, online_finetune=False, incre_bounds=False, + seed=None, + noisy_method=None, + parameter=None, + robust_loss=None, + device='cpu' ): """Experimental tool for the planar robotic arm experiments.""" @@ -633,36 +656,73 @@ def arm_main( outdir=outdir, log_freq=log_freq, log_arch_freq=log_arch_freq, - seed=trial_id, + seed=seed, use_dis_embed=use_dis_embed, n_pref_data=n_pref_data, online_finetune=online_finetune, incre_bounds=incre_bounds, + noisy_method=noisy_method, + parameter=parameter, + robust_loss=robust_loss, + device=device ) if __name__ == "__main__": - if len(sys.argv) > 1: - trial_id = int(sys.argv[1]) + parser = argparse.ArgumentParser( + description="Run robot arm experiments") + parser.add_argument('--trial_id', type=int, default=0) + parser.add_argument('--noisy_method', type=str, choices=['stochastic', 'add_equal_noise', + 'flip_by_distance', 'flip_labels_asymmetric', 'noisy_labels_exact'], required=True) + parser.add_argument('--parameter', type=float, required=True) + parser.add_argument('--robust_loss',type=str,required=True) + parser.add_argument('--seed', type=int, required=True) + parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') + parser.add_argument('--cuda_index', type=int, default=0) + + args = parser.parse_args() + + if args.device == 'cuda': + if torch.cuda.is_available(): + device = torch.device(f'cuda:{args.cuda_index}') + else: + print("⚠️ CUDA was requested but is not available. Falling back to CPU.") + device = torch.device('cpu') else: - trial_id = 0 + device = torch.device('cpu') # QD-GT - arm_main(trial_id, method="qd") + # arm_main(trial_id, method="qd") - # AURORA - for method in ["pca", "ae"]: - for online_finetune in [True, False]: - arm_main(trial_id, method=method, online_finetune=online_finetune) + # # AURORA + # for method in ["pca", "ae"]: + # for online_finetune in [True, False]: + # arm_main(trial_id, method=method, online_finetune=online_finetune) # QDHF n_pref_data = 200 - for online_finetune in [False, True]: + out_dir = f'logs/{args.robust_loss}_logs' + os.makedirs(out_dir, exist_ok=True) + + for online_finetune in [True]: # False, data = n_pref_data if not online_finetune else n_pref_data // 4 arm_main( - trial_id, + trial_id=args.trial_id, + outdir=out_dir, method="qdhf", use_dis_embed=True, n_pref_data=data, online_finetune=online_finetune, + noisy_method=args.noisy_method, + parameter=args.parameter, + seed=args.seed, + robust_loss=args.robust_loss, + device=device ) + + + if args.device == 'cuda': + torch.cuda.empty_cache() + gc.collect() + + sys.exit(0) \ No newline at end of file diff --git a/maze/ribs/__init__.py b/maze/ribs/__init__.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/__init__.py b/maze/ribs/archives/__init__.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/_add_status.py b/maze/ribs/archives/_add_status.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/_archive_base.py b/maze/ribs/archives/_archive_base.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/_cvt_archive.py b/maze/ribs/archives/_cvt_archive.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/_grid_archive.py b/maze/ribs/archives/_grid_archive.py old mode 100644 new mode 100755 diff --git a/maze/ribs/archives/_sliding_boundaries_archive.py b/maze/ribs/archives/_sliding_boundaries_archive.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/__init__.py b/maze/ribs/emitters/__init__.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_emitter_base.py b/maze/ribs/emitters/_emitter_base.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_gaussian_emitter.py b/maze/ribs/emitters/_gaussian_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_gradient_emitter.py b/maze/ribs/emitters/_gradient_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_gradient_improvement_emitter.py b/maze/ribs/emitters/_gradient_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_improvement_emitter.py b/maze/ribs/emitters/_improvement_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_iso_line_emitter.py b/maze/ribs/emitters/_iso_line_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_optimizing_emitter.py b/maze/ribs/emitters/_optimizing_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/_random_direction_emitter.py b/maze/ribs/emitters/_random_direction_emitter.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/opt/__init__.py b/maze/ribs/emitters/opt/__init__.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/opt/_adam.py b/maze/ribs/emitters/opt/_adam.py old mode 100644 new mode 100755 diff --git a/maze/ribs/emitters/opt/_cma_es.py b/maze/ribs/emitters/opt/_cma_es.py old mode 100644 new mode 100755 diff --git a/maze/ribs/factory.py b/maze/ribs/factory.py old mode 100644 new mode 100755 diff --git a/maze/ribs/optimizers/__init__.py b/maze/ribs/optimizers/__init__.py old mode 100644 new mode 100755 diff --git a/maze/ribs/optimizers/_optimizer.py b/maze/ribs/optimizers/_optimizer.py old mode 100644 new mode 100755 diff --git a/maze/ribs/visualize.py b/maze/ribs/visualize.py old mode 100644 new mode 100755 diff --git a/maze/robust_loss.py b/maze/robust_loss.py new file mode 100644 index 0000000..33466af --- /dev/null +++ b/maze/robust_loss.py @@ -0,0 +1,163 @@ +import torch +import torch.nn as nn + +class RobustLossAgent: + def __init__(self, margin=0.05): + self.margin = margin + + def robust_loss(self, delta_dis, y, robust_loss, parameter, robust_parameter=None, epoch=0): + robust_parameter = robust_parameter or {} + + if robust_loss == 'reweight': + beta = 0.2 + return self.reweighted_triplet_loss(delta_dis, y, beta=beta) + + elif robust_loss == 'truncated': + alpha = 0.05 + epsilon_max = 0.2 + return self.truncated_triplet_loss(delta_dis, y, epoch, alpha=alpha, epsilon_max=epsilon_max) + + elif robust_loss == 'label_smoothing': + alpha = 0.05 + return self.label_smoothing_triplet_loss(delta_dis, y, alpha=alpha) + + elif robust_loss == 'rDPO': + epsilon = parameter + return self.rDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'cDPO': + epsilon = parameter + return self.cDPO_triplet_loss(delta_dis, y, epsilon=epsilon) + + elif robust_loss == 'None': + loss_fn = lambda y, delta_dis: torch.max(torch.tensor([0.0]), 0.05 - y * delta_dis).mean() + return loss_fn(y, delta_dis) + + else: + raise ValueError(f"unknown robust method: {robust_loss}") + + def reweighted_triplet_loss(self, delta_dis, y, beta=0.2, eps=1e-6): + pred = torch.sigmoid(delta_dis) + weight = pred ** beta + raw_loss = weight * torch.clamp(self.margin - y * delta_dis, min=0.0) + normalized_loss = raw_loss.sum() * (delta_dis.size(0) / (weight.sum() + eps)) + return normalized_loss + + def truncated_triplet_loss(self, delta_dis, y, epoch, alpha=0.1, epsilon_max=0.1): + loss_sample = torch.clamp(self.margin - y * delta_dis, min=0.0) + drop_rate = min(alpha * epoch, epsilon_max) + batch_size = loss_sample.size(0) + + if drop_rate > 0: + kth = int(batch_size * (1 - drop_rate)) + if kth <= 0: + loss_sample = torch.zeros_like(loss_sample) + else: + tau = torch.kthvalue(loss_sample, kth).values + keep_mask = (loss_sample <= tau) + loss_sample = loss_sample * keep_mask.float() + + return loss_sample.mean() + + + def smooth_labels(self, y, alpha=0.1, num_classes=2): + if alpha <= 1e-6: + return y, y # 不平滑 + + y_idx = ((y + 1) / 2).long() + I = torch.eye(num_classes, device=y.device, dtype=y.dtype) + J = torch.ones((num_classes, num_classes), device=y.device, dtype=y.dtype) + M = (1 - alpha) * I + (alpha / num_classes) * J + + soft_labels = M[y_idx] + y_smooth = soft_labels[:, 1] - soft_labels[:, 0] # 转换为 ∈ [-1, 1] + y_smooth = torch.clamp(y_smooth, -1.0, 1.0) + return soft_labels, y_smooth + + + def label_smoothing_triplet_loss(self, delta_dis, y, alpha): + _, y_smooth = self.smooth_labels(y, alpha=alpha, num_classes=2) + loss = torch.clamp(self.margin - y_smooth * delta_dis, min=0.0) + return loss.mean() + + + def rDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + # print(epsilon) + y = y.to(delta_dis.device) + loss_clean = -torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + loss_flipped = -torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + numerator = (1 - epsilon) * loss_clean - epsilon * loss_flipped + denominator = 1 - 2 * epsilon + corrected_loss = numerator / (denominator + 1e-8) # add epsilon to avoid div 0 + return corrected_loss.mean() + + def cDPO_triplet_loss(self, delta_dis, y, epsilon=0.1): + y = y.to(delta_dis.device) + log_prob_clean = torch.log(torch.sigmoid(delta_dis * y) + 1e-8) # as if label = +1 + log_prob_flipped = torch.log(torch.sigmoid(-delta_dis * y) + 1e-8) # as if label = -1 + + # Weighted sum per noisy label distribution + loss = -((1 - epsilon) * log_prob_clean + epsilon * log_prob_flipped) + return loss.mean() + + + +class TripletCDRP(nn.Module): + def __init__(self, gamma=5.0, eta=0.5, eps=0.01): + super(TripletCDRP, self).__init__() + self.gamma = gamma + self.eta = eta + self.eps = eps + + def forward(self, ref_embed, x1_embed, x2_embed, pref_label): + d1 = torch.sum((ref_embed - x1_embed) ** 2, dim=-1) + d2 = torch.sum((ref_embed - x2_embed) ** 2, dim=-1) + logits = torch.stack([-d1, -d2], dim=1) # (B, 2) + logits = torch.clamp(logits, min=-20, max=20) + probs = torch.softmax(logits, dim=1) # (B, 2) + + labels = ((pref_label + 1) // 2).long() + + # Base CE loss + ce_loss = nn.NLLLoss(reduction="none") + log_probs = torch.log(probs + 1e-8) + # nominal_loss = ce_loss(log_probs, labels) # (B,) + + # C(y', y) = gamma if y' != y else 0 + gamma = self.gamma + penalty = gamma * (1 - torch.nn.functional.one_hot(labels, num_classes=2).to(logits.device)) + + # robust loss inner: max_y' [l(y') - gamma * 1_{y' ≠ y}] + loss_all = ce_loss(log_probs.repeat(1, 2).reshape(-1, 2), torch.tensor([0, 1] * logits.shape[0]).to(logits.device)).reshape(-1, 2) + robust_loss = (loss_all - penalty).max(dim=1).values + + final_loss = self.eps * gamma + robust_loss.mean() + + return final_loss + + +class InstanceEarlyStopper: + def __init__(self, num_samples, patience=3, delta=1e-4): + self.patience = patience + self.delta = delta + self.loss_history = [[] for _ in range(num_samples)] + self.mask = torch.zeros(num_samples, dtype=torch.bool) # True 表示“停止训练该样本” + + def update(self, indices, losses): + for i, idx in enumerate(indices): + idx = int(idx) + if self.mask[idx]: + continue + self.loss_history[idx].append(losses[i].item()) + + if len(self.loss_history[idx]) >= self.patience: + l_hist = self.loss_history[idx][-self.patience:] + if len(l_hist) == self.patience: + delta2 = sum([-l_hist[j] + 2*l_hist[j+1] - l_hist[j+2] for j in range(self.patience - 2)]) + + if abs(delta2) < self.delta: + self.mask[idx] = True + + def get_active_indices(self, all_indices): + return [idx for idx in all_indices if not self.mask[int(idx)]] diff --git a/maze/run_experiment.py b/maze/run_experiment.py new file mode 100755 index 0000000..0651669 --- /dev/null +++ b/maze/run_experiment.py @@ -0,0 +1,120 @@ +import subprocess +import pandas as pd +import numpy as np +import re +import os +import gc +import time +import torch +import argparse + +parser = argparse.ArgumentParser(description="Run QDHF experiments with config") +parser.add_argument("--noise_type", type=str, required=True) +parser.add_argument("--robust", type=str) +parser.add_argument('--device', type=str, choices=['cpu', 'cuda'], default='cpu') +args = parser.parse_args() + +# 实验设置 +noisy_list = { + "noisy_labels_exact": [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], # 0.05, 0.1, 0.2, + "stochastic": [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], + "add_equal_noise": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_by_distance": [1, 2, 5, 10, 15, 20, 25, 30], + "flip_labels_asymmetric": [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] +} +noisy_methods = {args.noise_type: noisy_list[args.noise_type]} +base_trial_ids = { + "stochastic": 10, + "add_equal_noise": 20, + "flip_by_distance": 30, + "flip_labels_asymmetric": 40, + "noisy_labels_exact": 50 +} +seeds = ["1111", "2222", "3333", "4444"] + +# 主逻辑 +for method, params in noisy_methods.items(): + csv_path = f"/mnt/data6t/qdhf/lsi/logs/{args.robust}_logs/{method}_experiment_results.csv" + + # 检查文件是否存在，如果存在则跳过创建 + if os.path.exists(csv_path): + print(f"CSV file for method {method} already exists, skipping creation.") + header_written = False + else: + header_written = True + + base_trial_id = base_trial_ids[method] + for i, param in enumerate(params): + trial_id = base_trial_id + i + + qd_scores = [] + coverage_scores = [] + + + for seed in seeds: + print(f"Running experiment with method={method}, param={param}, robust={args.robust}, trial_id={trial_id}, seed={seed}") + result = subprocess.run( + ["python", "main.py", "--seed", seed, "--trial_id", str(trial_id), + "--noisy_method", method, "--parameter", str(param), "--robust_loss", + args.robust, "--device", args.device], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + result.check_returncode() + output = result.stdout + print(output) + + try: + match = re.search(r"QD score: ([\d\.]+) Coverage: ([\d\.]+)", output) + if match: + qd_score = float(match.group(1)) + coverage = float(match.group(2)) + else: + qd_score, coverage = np.nan, np.nan + except Exception: + qd_score, coverage = np.nan, np.nan + + qd_scores.append(qd_score) + coverage_scores.append(coverage) + + # 每次实验结果写入 CSV + record = { + "Method": method, + "Parameter": param, + "Seed": seed, + "Trial ID": trial_id, + "QD Score": qd_score, + "Coverage": coverage + } + + pd.DataFrame([record]).to_csv(csv_path, mode='a', header=header_written, index=False) + header_written = False # 只有第一次写入时才写表头 + + gc.collect() + time.sleep(2) + + qd_mean = np.nanmean(qd_scores) + qd_std = np.nanstd(qd_scores) + coverage_mean = np.nanmean(coverage_scores) + coverage_std = np.nanstd(coverage_scores) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "Mean", + "Trial ID": "Mean", + "QD Score": qd_mean, + "Coverage": coverage_mean + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + + stats_record = { + "Method": method, + "Parameter": param, + "Seed": "STD", + "Trial ID": "STD", + "QD Score": qd_std, + "Coverage": coverage_std + } + pd.DataFrame([stats_record]).to_csv(csv_path, mode='a', header=False, index=False) + +print(f"Experiment results have been saved to {csv_path}") diff --git a/maze/run_maze.sh b/maze/run_maze.sh new file mode 100755 index 0000000..dafb11c --- /dev/null +++ b/maze/run_maze.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export CUDA_VISIBLE_DEVICES=7 + +python run_experiment.py --noise_type noisy_labels_exact --robust reweight --device cpu \ No newline at end of file diff --git a/maze/run_simple.sh b/maze/run_simple.sh new file mode 100755 index 0000000..d0bcccb --- /dev/null +++ b/maze/run_simple.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +export CUDA_VISIBLE_DEVICES=6 + +python main.py --seed 2222 --trial_id 51 --noisy_method noisy_labels_exact --parameter 0.05 --robust_loss None --device cpu --cuda_index 0 \ No newline at end of file diff --git a/maze/utils.py b/maze/utils.py old mode 100644 new mode 100755 index 73e46f0..ab6dfb1 --- a/maze/utils.py +++ b/maze/utils.py @@ -3,7 +3,8 @@ from torch import nn import numpy as np import time - +from generate_noise import NoiseGenerator +from robust_loss import RobustLossAgent, TripletCDRP, InstanceEarlyStopper class DisEmbed(nn.Module): def __init__(self, input_dim, latent_dim): @@ -34,8 +35,11 @@ def triplet_delta_dis(self, ref, x1, x2): def fit_dis_embed( - inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu" + inputs, gt_measures, latent_dim, batch_size=32, seed=None, device="cpu", noisy_method=None, parameter=None, robust_loss=None ): + # 这个函数使用 triplet-based contrastive learning，训练一个模型在嵌入空间中表示“人类感知下的多样性” + # inputs是随机产生的角度 + # print(device) t = time.time() inputs = np.array(inputs) gt_measures = np.array(gt_measures) @@ -44,7 +48,7 @@ def fit_dis_embed( optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) loss_fn = lambda y, delta_dis: torch.max( torch.tensor([0.0]), 0.05 - y * delta_dis - ).mean() + ).mean() # hinge triplet loss n_pref_data = inputs.shape[0] ref = inputs[:, 0] x1 = inputs[:, 1] @@ -78,31 +82,98 @@ def fit_dis_embed( # x1_gt_val = x1_gt_train # x2_gt_val = x2_gt_train + # if robust_loss == 'ies': + # early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=3, delta=1e-3) # best_acc = 0 # counter = 0 val_acc = [] for epoch in range(1000): - for _ in range(n_iters_per_epoch): - idx = np.random.choice(n_train, batch_size) - batch_ref = torch.tensor(ref_train[idx], dtype=torch.float32).to(device) - batch1 = torch.tensor(x1_train[idx], dtype=torch.float32).to(device) - batch2 = torch.tensor(x2_train[idx], dtype=torch.float32).to(device) + if epoch < 100: + early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=3, delta=1e-1) + else: + early_stopper = InstanceEarlyStopper(num_samples=n_train, patience=3, delta=5e-2) + if robust_loss == 'ies': + full_idx = np.arange(n_train) + active_idx = early_stopper.get_active_indices(full_idx) + + if len(active_idx) <= batch_size: + print("All triplets stopped early.") + break + + batch_idx = np.random.choice(active_idx, batch_size) + + batch_ref = torch.tensor(ref_train[batch_idx], dtype=torch.float32).to(device) + batch1 = torch.tensor(x1_train[batch_idx], dtype=torch.float32).to(device) + batch2 = torch.tensor(x2_train[batch_idx], dtype=torch.float32).to(device) + + ref_embed = model.forward(batch_ref) + x1_embed = model.forward(batch1) + x2_embed = model.forward(batch2) - optimizer.zero_grad() - delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) gt_dis = np.sum( - ( - np.square(ref_gt_train[idx] - x1_gt_train[idx]) - - np.square(ref_gt_train[idx] - x2_gt_train[idx]) - ), - -1, + np.square(ref_gt_train[batch_idx] - x1_gt_train[batch_idx]) - + np.square(ref_gt_train[batch_idx] - x2_gt_train[batch_idx]), + axis=-1 ) gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 + noise_gen = NoiseGenerator() + gt_noise = noise_gen.generate_noise(gt, gt_dis, noisy_method=noisy_method, parameter=parameter).to(device) + + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + loss_samplewise = torch.clamp(0.05 - gt_noise * delta_dis, min=0.0) # shape: (B,) + + # if epoch <= 3: + # print(f"Mean: {loss_samplewise.mean().item()}") + # print(f"Standard Deviation: {loss_samplewise.std().item()}") + # print(f"Max: {loss_samplewise.max().item()}") + # print(f"Min: {loss_samplewise.min().item()}") + + loss = loss_samplewise.mean() - loss = loss_fn(gt, delta_dis) + optimizer.zero_grad() loss.backward() optimizer.step() + # 更新 IES 模块 + early_stopper.update(batch_idx, loss_samplewise.detach().cpu()) + else: + for _ in range(n_iters_per_epoch): + idx = np.random.choice(n_train, batch_size) + batch_ref = torch.tensor(ref_train[idx], dtype=torch.float32).to(device) + batch1 = torch.tensor(x1_train[idx], dtype=torch.float32).to(device) + batch2 = torch.tensor(x2_train[idx], dtype=torch.float32).to(device) + + optimizer.zero_grad() + + gt_dis = np.sum( + ( + np.square(ref_gt_train[idx] - x1_gt_train[idx]) + - np.square(ref_gt_train[idx] - x2_gt_train[idx]) + ), + -1, + ) + # print("abs(gt_dis) percentiles:",np.percentile(np.abs(gt_dis), [0, 5, 10, 20, 30, 50, 60, 70, 90, 100])) + noise_gen = NoiseGenerator() + gt = torch.tensor(gt_dis > 0, dtype=torch.float32) * 2 - 1 # 产生无噪声标签 + gt_noise = noise_gen.generate_noise( + gt, gt_dis, noisy_method=noisy_method, parameter=parameter) + + # loss = loss_fn(gt_noise, delta_dis) + + # 使用 Reweighted loss + if robust_loss == 'crdo': + ref_embed = model.forward(batch_ref).to(device) + x1_embed = model.forward(batch1).to(device) + x2_embed = model.forward(batch2).to(device) + triplet_loss_fn = TripletCDRP(gamma=5.0, eta=0.5, eps=0.01).to(device) + loss = triplet_loss_fn(ref_embed, x1_embed, x2_embed, gt_noise).to(device) + else: + loss_agent = RobustLossAgent(margin=0.05) + delta_dis = model.triplet_delta_dis(batch_ref, batch1, batch2) + loss = loss_agent.robust_loss(delta_dis, gt_noise, robust_loss, parameter, epoch, device).to(device) + loss.backward() + optimizer.step() + # Evaluate. n_correct = 0 n_total = 0 diff --git a/requirement_nips.txt b/requirement_nips.txt new file mode 100644 index 0000000..b565075 --- /dev/null +++ b/requirement_nips.txt @@ -0,0 +1,130 @@ +about-time==4.2.1 +absl-py==2.2.2 +accelerate==1.6.0 +alive-progress==3.1.1 +attrs==25.3.0 +blinker==1.9.0 +brax==0.12.1 +certifi==2025.1.31 +charset-normalizer==3.4.1 +chex==0.1.89 +click==8.1.8 +clip @ git+https://github.com/openai/CLIP.git@dcba3cb2e2827b402d2701e7e1c7d9fed8a20ef1 +cloudpickle==3.1.1 +contourpy==1.3.1 +cycler==0.12.1 +decorator==5.2.1 +diffusers==0.32.2 +dm-env==1.6 +dm-tree==0.1.9 +dreamsim==0.2.1 +etils==1.12.2 +Farama-Notifications==0.0.4 +filelock==3.18.0 +Flask==3.1.0 +flask-cors==5.0.1 +flax==0.10.5 +fonttools==4.57.0 +fsspec==2025.3.2 +ftfy==6.3.1 +gast==0.6.0 +glfw==2.8.0 +grapheme==0.6.0 +grpcio==1.71.0 +gym==0.26.2 +gym-notices==0.0.8 +gymnasium==1.1.1 +huggingface-hub==0.30.1 +humanize==4.12.2 +idna==3.10 +importlib_metadata==8.6.1 +importlib_resources==6.5.2 +itsdangerous==2.2.0 +jax==0.5.3 +jax-cuda12-pjrt==0.5.3 +jax-cuda12-plugin==0.5.3 +jaxlib==0.5.3 +jaxopt==0.8.3 +Jinja2==3.1.6 +joblib==1.4.2 +jumanji==1.1.0 +kiwisolver==1.4.8 +llvmlite==0.44.0 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.7.5 +mdurl==0.1.2 +ml_collections==1.0.0 +ml_dtypes==0.5.1 +mpmath==1.3.0 +msgpack==1.1.0 +mujoco==3.3.0 +mujoco-mjx==3.3.0 +nest-asyncio==1.6.0 +networkx==3.4.2 +numba==0.61.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvcc-cu12==12.8.93 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +open_clip_torch==2.32.0 +opt_einsum==3.4.0 +optax==0.2.4 +orbax-checkpoint==0.11.10 +packaging==24.2 +pandas==2.2.3 +peft==0.15.1 +pillow==11.1.0 +protobuf==6.30.2 +psutil==7.0.0 +Pygments==2.19.1 +PyOpenGL==3.1.9 +pyparsing==3.2.3 +python-dateutil==2.9.0.post0 +pytinyrenderer==0.0.14 +pytz==2025.2 +PyYAML==6.0.2 +qdax==0.4.1 +regex==2024.11.6 +requests==2.32.3 +rich==14.0.0 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.2 +simplejson==3.20.1 +six==1.17.0 +sortedcontainers==2.4.0 +sympy==1.13.1 +tensorboardX==2.6.2.2 +tensorflow-probability==0.25.0 +tensorstore==0.1.73 +threadpoolctl==3.6.0 +timm==1.0.15 +tokenizers==0.21.1 +toml==0.10.2 +toolz==1.0.0 +torch==2.6.0 +torchvision==0.21.0 +tqdm==4.67.1 +transformers==4.51.0 +treescope==0.1.9 +trimesh==4.6.6 +triton==3.2.0 +typing_extensions==4.13.1 +tzdata==2025.2 +urllib3==2.3.0 +wcwidth==0.2.13 +Werkzeug==3.1.3 +wrapt==1.17.2 +zipp==3.21.0