rust-vs-python-ml-bench/nextflow.config at main · RantAI-dev/rust-vs-python-ml-bench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
params {
    // Input/Output directories
    data_dir = './data'
    results_dir = './results'
    config_dir = './config'

    // Benchmark configuration
    benchmarks_config = "${config_dir}/benchmarks.yaml"
    frameworks_config = "${config_dir}/frameworks.yaml"
    hardware_config = "${config_dir}/hardware.yaml"

    // Execution parameters
    repetitions = 10
    confidence_level = 0.95
    enable_gpu = true
    enable_monitoring = true

    // Container settings
    container_registry = 'ghcr.io/rust-ml-benchmark'
    python_container = "${container_registry}/python-ml-benchmark:latest"
    rust_container = "${container_registry}/rust-ml-benchmark:latest"

    // Resource limits
    max_memory = '32.GB'
    max_cpus = 16
    max_time = '24.h'
}

// Execution profiles
profiles {
    local {
        executor.name = 'local'
        executor.cpus = 8
        executor.memory = '16 GB'
    }

    slurm {
        executor.name = 'slurm'
        executor.queueSize = 50
        process.queue = 'gpu'
        process.clusterOptions = '--gres=gpu:1'
    }

    docker {
        docker.enabled = true
        docker.userEmulation = true
        singularity.enabled = false
    }

    singularity {
        singularity.enabled = true
        singularity.autoMounts = true
        docker.enabled = false
    }

    local_smoke {
        // Minimal local profile for quick smoke runs
        executor.name = 'local'
        process.executor = 'local'
        docker.enabled = false
        singularity.enabled = false
        params.repetitions = 1
        params.enable_gpu = false
        params.enable_monitoring = true
        process.cpus = 4
        process.memory = '6 GB'

        process {
            withLabel: 'cpu_intensive' {
                cpus = 1
                memory = '2 GB'
                time = '1.h'
            }
            withLabel: 'gpu_training' {
                cpus = 1
                memory = '2 GB'
                time = '1.h'
            }
            withLabel: 'memory_intensive' {
                cpus = 1
                memory = '2 GB'
                time = '1.h'
            }
        }
    }
}

// Process resource configuration
process {
    withLabel: 'cpu_intensive' {
        cpus = 1
        memory = '2 GB'
        time = '2.h'
    }

    withLabel: 'gpu_training' {
        cpus = 2
        memory = '4 GB'
        time = '6.h'
        clusterOptions = '--gres=gpu:1'
    }

    withLabel: 'memory_intensive' {
        cpus = 1
        memory = '8 GB'
        time = '4.h'
    }

    withLabel: 'smoke_light' {
        cpus = 1
        memory = '2.GB'
        time = '1.h'
    }
}