From 4bff0e67a618be7d2ffbca2b89909def6538208e Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 12 May 2026 13:30:34 +0200 Subject: [PATCH 1/2] move conda_auto_init under pulsar_yaml_config, where it belongs - rename vars that are not related to pulsar role - remove unused vars --- README.md | 2 +- group_vars/pulsarservers.yml | 26 +++++++++----------- host_vars/pulsar-re2.grid.cesnet.cz/vars.yml | 4 +-- pulsar.yml | 8 +++--- roles/metacentrum.krbnfs/tasks/main.yml | 6 ++--- roles/metacentrum.pulsar/tasks/main.yml | 21 ++++++++-------- 6 files changed, 32 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 45ec0a6..19a346c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ pulsar: nfs_home: "brno11-elixir" nfs_prefix: pulsar-qa2 -install_nfs_conda: false +csnt_install_nfs_conda: false rabbitmq_hostname: "galaxy-qa2.galaxy.cloud.e-infra.cz" rabbitmq_vhost: "pulsar" diff --git a/group_vars/pulsarservers.yml b/group_vars/pulsarservers.yml index d7a22b9..bb0a9e4 100644 --- a/group_vars/pulsarservers.yml +++ b/group_vars/pulsarservers.yml @@ -1,11 +1,10 @@ -drmaa_version: 1.0.19 -pbs_server: pbs-m1.metacentrum.cz -install_nfs_conda: false -conda_auto_init: "{{ install_nfs_conda }}" -pulsar_network_homedir: "/storage/{{ pulsar.nfs_home }}/home/{{ pulsar.user_name }}" +csnt_drmaa_version: 1.0.19 +csnt_pbs_server: pbs-m1.metacentrum.cz +csnt_install_nfs_conda: false +csnt_pulsar_network_homedir: "/storage/{{ pulsar.nfs_home }}/home/{{ pulsar.user_name }}" # Miniconda settings -miniconda_prefix: "{{ pulsar_network_homedir }}/conda" +miniconda_prefix: "{{ csnt_pulsar_network_homedir }}/conda" miniconda_exec: "{{ miniconda_prefix }}/bin/mamba" miniconda_version: 23.1.0 miniconda_channels: @@ -17,16 +16,14 @@ csnt_dependency_resolvers: {} pulsar_user: "{{ pulsar.user_name }}" pulsar_root: /opt/pulsar pulsar_package_version: "0.15.6" -pulsar_data_dir: "{{ pulsar_network_homedir }}/{{ pulsar.nfs_prefix }}" -pulsar_staging_dir: "{{ pulsar_data_dir }}/files/staging" -pulsar_cleanup_dir: "{{ pulsar_data_dir }}/files/cleanup_jobs" -pulsar_cleanup_delay: "7d" # DEMON: pulsar-re needs this to have over 30d +csnt_pulsar_data_dir: "{{ csnt_pulsar_network_homedir }}/{{ pulsar.nfs_prefix }}" +pulsar_staging_dir: "{{ csnt_pulsar_data_dir }}/files/staging" +csnt_pulsar_cleanup_dir: "{{ csnt_pulsar_data_dir }}/files/cleanup_jobs" +csnt_pulsar_cleanup_delay: "7d" # DEMON: pulsar-re needs this to have over 30d pulsar_persistence_dir: "{{ pulsar_root }}/files/persistent" -pulsar_dependencies_dir: "{{ pulsar_data_dir }}/deps" -pulsar_singularity_cachedir: "{{ pulsar_network_homedir }}/.singularity" -pulsar_package_change_dir: "{{ pulsar_root }}/venv/lib/python3.11/site-packages/pulsar/client" +pulsar_dependencies_dir: "{{ csnt_pulsar_data_dir }}/deps" +csnt_pulsar_package_change_dir: "{{ pulsar_root }}/venv/lib/python3.11/site-packages/pulsar/client" pulsar_separate_privileges: true -pulsar_pip_install: true pulsar_pycurl_ssl_library: openssl pulsar_systemd: true pulsar_systemd_enabled: true @@ -52,6 +49,7 @@ pulsar_optional_dependencies: - psutil - pylockfile pulsar_yaml_config: + conda_auto_init: "{{ csnt_install_nfs_conda }}" staging_directory: "{{ pulsar_staging_dir }}" persistence_directory: "{{ pulsar_persistence_dir }}" tool_dependency_dir: "{{ pulsar_dependencies_dir }}" diff --git a/host_vars/pulsar-re2.grid.cesnet.cz/vars.yml b/host_vars/pulsar-re2.grid.cesnet.cz/vars.yml index 4ab40c5..fc214b6 100644 --- a/host_vars/pulsar-re2.grid.cesnet.cz/vars.yml +++ b/host_vars/pulsar-re2.grid.cesnet.cz/vars.yml @@ -7,14 +7,14 @@ pulsar: nfs_home: "brno11-elixir" nfs_prefix: pulsar-re -miniconda_channels: +miniconda_channels: - 'bioconda' - 'conda-forge' - 'petrnovak' miniconda_prefix: "/storage/{{ pulsar.nfs_home }}/home/{{ pulsar.user_name }}/anaconda3" miniconda_exec: "{{ miniconda_prefix }}/bin/mamba" -pulsar_cleanup_delay: "40d" # DEMON: pulsar-re* need this to have over 30d +csnt_pulsar_cleanup_delay: "40d" # DEMON: pulsar-re* need this to have over 30d csnt_dependency_resolvers: - type: conda auto_init: true diff --git a/pulsar.yml b/pulsar.yml index 6b6d00c..a5d3a7b 100644 --- a/pulsar.yml +++ b/pulsar.yml @@ -37,7 +37,7 @@ - name: Apply recursive rsync changes in pulsar package ansible.posix.patch: src: "files/pulsar_repo/{{ pulsar_package_version }}/recursive_rsync.patch" - basedir: "{{ pulsar_package_change_dir }}" + basedir: "{{ csnt_pulsar_package_change_dir }}" backup: true strip: 1 - name: Restart pulsar service after change @@ -48,13 +48,13 @@ - name: Create .drmaa folder in NFS home ansible.builtin.file: - path: "{{ pulsar_network_homedir }}/.drmaa" + path: "{{ csnt_pulsar_network_homedir }}/.drmaa" state: directory become: yes become_user: "{{ pulsar.user_name }}" - name: Create cleanup_jobs dir ansible.builtin.file: - path: "{{ pulsar_cleanup_dir }}" + path: "{{ csnt_pulsar_cleanup_dir }}" state: directory recurse: yes mode: 'a+x' @@ -65,6 +65,6 @@ name: "tmpreaper in staging dir" minute: "7" hour: "*/6" - job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 {{ pulsar_cleanup_delay }} {{ pulsar_staging_dir }} >> {{ pulsar_cleanup_dir }}/cleaning_pulsar_files.log" + job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 {{ csnt_pulsar_cleanup_delay }} {{ pulsar_staging_dir }} >> {{ csnt_pulsar_cleanup_dir }}/cleaning_pulsar_files.log" become: yes become_user: "{{ pulsar.user_name }}" diff --git a/roles/metacentrum.krbnfs/tasks/main.yml b/roles/metacentrum.krbnfs/tasks/main.yml index 8ff8b51..aaaebb4 100644 --- a/roles/metacentrum.krbnfs/tasks/main.yml +++ b/roles/metacentrum.krbnfs/tasks/main.yml @@ -6,7 +6,7 @@ - krb5-user - anacron - logrotate - + - name: Metacentrum repo key apt_key: url: https://repo.metacentrum.cz/key.asc @@ -29,12 +29,12 @@ - nfs-common - name: PBS config - lineinfile: + lineinfile: path: /etc/pbs.conf regexp: "^{{ item.key }}=" line: "{{ item.key }}={{ item.value }}" loop: - - { key: 'PBS_SERVER', value: '{{ pbs_server }}' } + - { key: 'PBS_SERVER', value: '{{ csnt_pbs_server }}' } - { key: 'PBS_AUTH_METHOD', value: 'GSS' } - { key: 'PBS_ENCRYPT_METHOD', value: 'GSS' } - { key: 'PBS_SUPPORTED_AUTH_METHODS', value: 'GSS,resvport' } diff --git a/roles/metacentrum.pulsar/tasks/main.yml b/roles/metacentrum.pulsar/tasks/main.yml index 5574bb7..9809b3b 100644 --- a/roles/metacentrum.pulsar/tasks/main.yml +++ b/roles/metacentrum.pulsar/tasks/main.yml @@ -23,15 +23,15 @@ - name: Download DRMAA get_url: - url: https://master.dl.sourceforge.net/project/pbspro-drmaa/pbs-drmaa/1.0/pbs-drmaa-{{ drmaa_version }}.tar.gz?viasf=1 - dest: /tmp/pbs-drmaa-{{ drmaa_version }}.tar.gz + url: https://master.dl.sourceforge.net/project/pbspro-drmaa/pbs-drmaa/1.0/pbs-drmaa-{{ csnt_drmaa_version }}.tar.gz?viasf=1 + dest: /tmp/pbs-drmaa-{{ csnt_drmaa_version }}.tar.gz - name: Install DRMAA shell: | cd /tmp - rm -rf pbs-drmaa-{{ drmaa_version }} - tar xzf pbs-drmaa-{{ drmaa_version }}.tar.gz - cd pbs-drmaa-{{ drmaa_version }} + rm -rf pbs-drmaa-{{ csnt_drmaa_version }} + tar xzf pbs-drmaa-{{ csnt_drmaa_version }}.tar.gz + cd pbs-drmaa-{{ csnt_drmaa_version }} ./configure --prefix /opt/drmaa make && make install @@ -72,13 +72,13 @@ dest: "/home/{{ pulsar.user_name }}/Miniconda3-py39_{{ miniconda_version }}-1-Linux-x86_64.sh" become: yes become_user: "{{ pulsar.user_name }}" - when: not miniconda_exists.stat.exists and install_nfs_conda + when: not miniconda_exists.stat.exists and csnt_install_nfs_conda - name: Install Miniconda to NFS home ansible.builtin.shell: cmd: "/bin/bash /home/{{ pulsar.user_name }}/Miniconda3-py39_{{ miniconda_version }}-1-Linux-x86_64.sh -b -p {{ miniconda_prefix }}" become: yes become_user: "{{ pulsar.user_name }}" - when: not miniconda_exists.stat.exists and install_nfs_conda + when: not miniconda_exists.stat.exists and csnt_install_nfs_conda - name: Ensure conda folder is executable ansible.builtin.file: path: "{{ miniconda_prefix }}/bin" @@ -87,7 +87,7 @@ mode: 'a+x' become: yes become_user: "{{ pulsar.user_name }}" - when: install_nfs_conda + when: csnt_install_nfs_conda - name: Check for mamba bin stat: path: "{{ miniconda_prefix }}/bin/mamba" @@ -98,7 +98,7 @@ ansible.builtin.command: "{{ miniconda_prefix }}/bin/conda install -y -c conda-forge mamba" become: yes become_user: "{{ pulsar.user_name }}" - when: not mamba_exists.stat.exists and install_nfs_conda + when: not mamba_exists.stat.exists and csnt_install_nfs_conda - name: Ensure conda folder is executable again ansible.builtin.file: path: "{{ miniconda_prefix }}/bin" @@ -107,5 +107,4 @@ mode: 'a+x' become: yes become_user: "{{ pulsar.user_name }}" - when: not mamba_exists.stat.exists and install_nfs_conda - + when: not mamba_exists.stat.exists and csnt_install_nfs_conda From c3b1cfc475ccb3afa14370dc426d281e7c4d2fc6 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 12 May 2026 13:41:40 +0200 Subject: [PATCH 2/2] move csnt_ prefixed vars to a group --- group_vars/pulsarservers.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/group_vars/pulsarservers.yml b/group_vars/pulsarservers.yml index bb0a9e4..c417b2f 100644 --- a/group_vars/pulsarservers.yml +++ b/group_vars/pulsarservers.yml @@ -2,6 +2,11 @@ csnt_drmaa_version: 1.0.19 csnt_pbs_server: pbs-m1.metacentrum.cz csnt_install_nfs_conda: false csnt_pulsar_network_homedir: "/storage/{{ pulsar.nfs_home }}/home/{{ pulsar.user_name }}" +csnt_dependency_resolvers: {} +csnt_pulsar_data_dir: "{{ csnt_pulsar_network_homedir }}/{{ pulsar.nfs_prefix }}" +csnt_pulsar_cleanup_dir: "{{ csnt_pulsar_data_dir }}/files/cleanup_jobs" +csnt_pulsar_cleanup_delay: "7d" # DEMON: pulsar-re needs this to have over 30d +csnt_pulsar_package_change_dir: "{{ pulsar_root }}/venv/lib/python3.11/site-packages/pulsar/client" # Miniconda settings miniconda_prefix: "{{ csnt_pulsar_network_homedir }}/conda" @@ -10,19 +15,14 @@ miniconda_version: 23.1.0 miniconda_channels: - 'bioconda' - 'conda-forge' -csnt_dependency_resolvers: {} # only galaxyproject.pulsar role vars below pulsar_user: "{{ pulsar.user_name }}" pulsar_root: /opt/pulsar pulsar_package_version: "0.15.6" -csnt_pulsar_data_dir: "{{ csnt_pulsar_network_homedir }}/{{ pulsar.nfs_prefix }}" pulsar_staging_dir: "{{ csnt_pulsar_data_dir }}/files/staging" -csnt_pulsar_cleanup_dir: "{{ csnt_pulsar_data_dir }}/files/cleanup_jobs" -csnt_pulsar_cleanup_delay: "7d" # DEMON: pulsar-re needs this to have over 30d pulsar_persistence_dir: "{{ pulsar_root }}/files/persistent" pulsar_dependencies_dir: "{{ csnt_pulsar_data_dir }}/deps" -csnt_pulsar_package_change_dir: "{{ pulsar_root }}/venv/lib/python3.11/site-packages/pulsar/client" pulsar_separate_privileges: true pulsar_pycurl_ssl_library: openssl pulsar_systemd: true