CriGrad/eval.py at main · heejbin/CriGrad · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import os
import argparse

def parse_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=1, help='GPU to use')
    parser.add_argument('--metric', type=str, default='', help='Metric to Eval')
    parser.add_argument('--k', type=float, default=-1, help='k value for experiment')
    parser.add_argument('--crigrad', action='store_true', help='Using CriGrad')
    parser.add_argument('--path', type =str, default = "default", help = "saliency map location")
    parser.add_argument('--max_data', type = int, default = -1, help = "number of data")
    parser.add_argument('--dataset', type=str, default='imagenet', help='Dataset')
    parser.add_argument('--model', type=str, default='resnet18', help='Model')
    args = parser.parse_args()

    return args

eval_saliency_methods = [
    'inputgrad',
    'GxI'
]

eval_metrics = [
    'pixel_perturbation',
    'sensitivity_n',
    'remove_and_retrain',
]

def run_eval(
    method,
    metric,
    crigrad,
    k=-1,
):
    if k == -1:
        k_list = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
    else:
        k_list = [k]

    if metric == 'pixel_perturbation':
        for k in k_list:
            cmd = f"python3 pixel_perturbation.py --method {method} --k {k} --path {args.path} --max_data {args.max_data} --dataset {args.dataset} --model {args.model}"
            if crigrad:
                cmd += ' --crigrad'
            print(cmd)
            os.system(cmd)
    elif metric == 'sensitivity_n':
        for k in k_list:
            cmd = f"python3 sensitivity_n.py --method {method} --q {k} --path {args.path} --max_data {args.max_data} --dataset {args.dataset} --model {args.model}"
            if crigrad:
                cmd += ' --crigrad'
            print(cmd)
            os.system(cmd)
    elif metric == 'remove_and_retrain':
        for k in k_list:
            for seed in range(5):
                cmd = f"python3 remove_and_retrain.py --method {method}  --k {k} --seed {seed} --path {args.path} --max_data {args.max_data} --dataset {args.dataset} --model {args.model}"
                if crigrad:
                    cmd += ' --crigrad'
                print(cmd)
                os.system(cmd)

if __name__ == "__main__":
    args = parse_arguments()

    for method in eval_saliency_methods:
        if len(args.metric) > 0:
            assert(args.metric in eval_metrics)
            run_eval(
                method,
                args.metric,
                args.crigrad,
                args.k
            )
        else:
            for metric in eval_metrics:
                run_eval(
                    method,
                    metric,
                    args.crigrad,
                    args.k
                )