From 44404522d488929c6a54c8157f063e5d4bedd74c Mon Sep 17 00:00:00 2001 From: keval nagda Date: Thu, 21 Mar 2019 13:41:28 +0530 Subject: [PATCH 1/3] Add yaml syntax validation --- .gitignore | 1 + simple_grid_yaml_compiler.py | 4 +- tests/output.yaml | 709 +++++++++++++++++++++++++++++++++++ tests/tests_semantics.py | 26 +- 4 files changed, 724 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 894a44c..66deb2e 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,4 @@ venv.bak/ # mypy .mypy_cache/ +.temp \ No newline at end of file diff --git a/simple_grid_yaml_compiler.py b/simple_grid_yaml_compiler.py index d0a525f..a5c38cc 100644 --- a/simple_grid_yaml_compiler.py +++ b/simple_grid_yaml_compiler.py @@ -49,7 +49,7 @@ def phase_5(phase_4_output, runtime_vars, yaml): if data_section == 'lightweight_components': updated_components = [] for lightweight_component in data[data_section]: - print "Component: " + lightweight_component['name'] + print ("Component: " + lightweight_component['name']) updated_component = {} for component_section in lightweight_component: if component_section == 'config': @@ -96,7 +96,7 @@ def phase_6(phase_5_output_file, yaml): container_split_config = yaml_augmentation.split_container_config(variable_hierarchies) #pass 4 with_ids = yaml_augmentation.add_execution_ids(container_split_config) - print with_ids + print (with_ids) yaml.dump(with_ids, phase_6_output_file) return phase_6_output_file diff --git a/tests/output.yaml b/tests/output.yaml index e69de29..063ce63 100644 --- a/tests/output.yaml +++ b/tests/output.yaml @@ -0,0 +1,709 @@ +meta_info_wn-pbs: + component: WN-Pbs + type: worker_node + version: 2.5.13 + docker_hub_url: '' + primary_config_key: pbs-info + site_level_config_version: 1.0 + default_var_prefix: pbs_default_var + runtime_vars: + - pbs_runtime_var_ce_host + host_requirements: + firewall: + - ports: 8080 + protocol: tcp + action: accept + ipv6: true + - ports: 6543 + protocol: tcp + action: accept + ipv6: false + cvmfs: true + docker_run_parameters: + privileged: true + mount: wn-pbs +default_group_opssgm: + gid: 46001 + name: opssgm +vars: +- 192.168.0.4 +- 192.168.0.5 +default_pool_accounts_alicesgm: + base_name: sgmali + initial_uid: 60101 + users_num: 10 + primary_group: + gid: 1397 + name: alicesgm + secondary_groups: + - gid: 1395 + name: alice +default_group_ops_name: ops +site: + name: simple-dev-site + email: mayank.sharma@cern.ch + latitude: 46.3 + longitude: 6.2 + location: CERN, Geneva, Switzerland + description: A demo site demonstrating the SIMPLE framework for site setup + website: https://home.cern + support_website: https://groups.google.com/forum/#!forum/wlcg-lightweight-sites + support_email: mayank.sharma@cern.ch + security_email: mayank.sharma@cern.ch + grid: wlcg + tier: 3 + bdii_host: bdii.cern.ch + use_argus: false +cream_runtime_variables: +- lightweight_component01.cern.ch +lightweight_components: +- lifecycle_hooks: + pre_config: + - /etc/simple_grid/lifecycle/ce_pre_config.sh + pre_init: + - /etc/simple_grid/lifecycle/ce_pre_inst1.sh + post_init: + - /etc/simple_grid/lifecycle/ce_post_inst1.sh + name: Cream-CE + deploy: + node: lightweight_component01.cern.ch + container_count: 1 + config: + ce_si00: 381 + cream_db_password: str() + cemon_host: lightweight_component01.cern.ch + ce_runtimeenv: EMI-3 + ce_close_se: SE3 + ce_cpu_vendor: Intel + ce_minvirtmem: 8192 + ce_close_se3_access_point: /not/used + batch_log_dir: /var/spool/pbs + apel_db_password: str() + ce_inboundip: false + blparser_with_updater_notifier: true + ce_logcpu: 0 + ce_os_arch: str() + apel_mysql_host: lightweight_component01.cern.ch + ce_otherdescr: Cores=4 + cream_db_user: str() + ce_sf00: 0 + ce_minphysmem: 2000 + ce_smpsize: 8 + ce_physcpu: 0 + ce_outboundip: true + se_list: not-used.some-domain + blparser_host: lightweight_component01.cern.ch + globus_tcp_port_range: + begin: 20000 + end: 21000 + ce_capability: CPUScalingReferenceSI00=1000 + mysql_password: fteu5746hg + ce_cpu_model: Xeon + ce_cpu_speed: 3000 + queues: + alice: + - voms_fqan: /alice + pool_accounts: + - base_name: alice + initial_uid: 10417 + users_num: 5 + primary_group: + gid: 1395 + name: alice + secondary_groups: [] + vo: + name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used + - voms_fqan: /alice/ROLE=lcgadmin + pool_accounts: + - base_name: alice + initial_uid: 10417 + users_num: 5 + primary_group: + gid: 1395 + name: alice + secondary_groups: [] + - base_name: sgmali + initial_uid: 60101 + users_num: 10 + primary_group: + gid: 1397 + name: alicesgm + secondary_groups: + - gid: 1395 + name: alice + vo: + name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used + ce_close_se3_host: not-used.some-domain + se_mount_info_list: none + repository_revision: master + supplemental_config: + some_additional_parameter: some_value + repository_url: https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream + type: compute_element + preferred_tech_stack: + level_2_configuration: yaim + container_count: 1 + id: 0 + execution_id: 0 +- lifecycle_hooks: + pre_config: + - /etc/simple_grid/lifecycle/wn_pre_config.sh + pre_init: + - /etc/simple_grid/lifecycle/wn_pre_inst1.sh + post_init: + - /etc/simple_grid/lifecycle/wn_post_inst1.sh + name: WN-Pbs + deploy: + node: lightweight_component02.cern.ch + container_count: 2 + config: + px_host: myproxy.cern.ch + batch_server: pbs + ce_smpsize: 8 + ce_close_se3_access_point: /not/used + ce_host: lightweight_component01.cern.ch + ce_close_se: SE3 + se_list: not-used.some-domain + ce_close_se3_host: not-used.some-domain + se_mount_info_list: none + repository_revision: master + supplemental_config: + some_additional_parameter: some_value + repository_url: https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_wn_pbs + type: worker_node + preferred_tech_stack: + level_2_configuration: yaim + container_count: 1 + id: 1 + execution_id: 1 +- lifecycle_hooks: + pre_config: + - /etc/simple_grid/lifecycle/wn_pre_config.sh + pre_init: + - /etc/simple_grid/lifecycle/wn_pre_inst1.sh + post_init: + - /etc/simple_grid/lifecycle/wn_post_inst1.sh + name: WN-Pbs + deploy: + node: lightweight_component02.cern.ch + container_count: 2 + config: + px_host: myproxy.cern.ch + batch_server: pbs + ce_smpsize: 8 + ce_close_se3_access_point: /not/used + ce_host: lightweight_component01.cern.ch + ce_close_se: SE3 + se_list: not-used.some-domain + ce_close_se3_host: not-used.some-domain + se_mount_info_list: none + repository_revision: master + supplemental_config: + some_additional_parameter: some_value + repository_url: https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_wn_pbs + type: worker_node + preferred_tech_stack: + level_2_configuration: yaim + container_count: 1 + id: 1 + execution_id: 2 +default_pool_accounts_alice: + base_name: alice + initial_uid: 10417 + users_num: 5 + primary_group: + gid: 1395 + name: alice + secondary_groups: [] +default_group_dteamsgm: + gid: 2680 + name: dteamsgm +preferred_tech_stack: + level_1_configuration: puppet + level_2_configuration: puppet + container_orchestration: docker-swarm + container: docker +default_pool_accounts_dteam: + base_name: dteam + initial_uid: 18118 + users_num: 10 + primary_group: + gid: 2688 + name: dteam + secondary_groups: [] +default_vo_dteam: + name: dteam + servers: + - server: voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/dteam +default_group_ops: + gid: 45000 + name: ops +default_group_alicesgm_name: alicesgm +pbs_runtime_variables: +- lightweight_component01.cern.ch +site_infrastructure: +- fqdn: lightweight_component01.cern.ch + ip_address: 192.168.0.4 +- fqdn: lightweight_component02.cern.ch + ip_address: 192.168.0.5 +default_pool_accounts_dteamsgm: + base_name: sgmdtm + initial_uid: 60501 + users_num: 5 + primary_group: + gid: 2680 + name: dteamsgm + secondary_groups: + - gid: 2688 + name: dteam +default_group_alice_name: alice +default_vo_generic_dn_ca_cern: /DC=ch/DC=cern/CN=CERN Grid Certification Authority +default_group_alicesgm: + gid: 1397 + name: alicesgm +supported_virtual_organizations: +- name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used +- name: dteam + servers: + - server: voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/dteam +- name: ops + servers: + - server: voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/ops +voms_config: +- voms_fqan: /alice + pool_accounts: + - base_name: alice + initial_uid: 10417 + users_num: 5 + primary_group: + gid: 1395 + name: alice + secondary_groups: [] + vo: + name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used +- voms_fqan: /alice/ROLE=lcgadmin + pool_accounts: + - base_name: alice + initial_uid: 10417 + users_num: 5 + primary_group: + gid: 1395 + name: alice + secondary_groups: [] + - base_name: sgmali + initial_uid: 60101 + users_num: 10 + primary_group: + gid: 1397 + name: alicesgm + secondary_groups: + - gid: 1395 + name: alice + vo: + name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used + comment: sgm +- voms_fqan: /dteam + pool_accounts: + - base_name: dteam + initial_uid: 18118 + users_num: 10 + primary_group: + gid: 2688 + name: dteam + secondary_groups: [] + vo: + name: dteam + servers: + - server: voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/dteam +- voms_fqan: /dteam/ROLE=lcgadmin + pool_accounts: + - base_name: sgmdtm + initial_uid: 60501 + users_num: 5 + primary_group: + gid: 2680 + name: dteamsgm + secondary_groups: + - gid: 2688 + name: dteam + - base_name: dteam + initial_uid: 18118 + users_num: 10 + primary_group: + gid: 2688 + name: dteam + secondary_groups: [] + vo: + name: dteam + servers: + - server: voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15001' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/dteam + comment: sgm +- voms_fqan: /ops + vo: + name: ops + servers: + - server: voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/ops + pool_accounts: + - base_name: ops + initial_uid: 45001 + users_num: 10 + primary_group: + gid: 45000 + name: ops + secondary_groups: [] +- voms_fqan: /ops/ROLE=lcgadmin + pool_accounts: + - base_name: sgmops + initial_uid: 60701 + users_num: 10 + primary_group: + gid: 45000 + name: ops + secondary_groups: + - gid: 46001 + name: opssgm + - base_name: ops + initial_uid: 45001 + users_num: 10 + primary_group: + gid: 45000 + name: ops + secondary_groups: [] + vo: + name: ops + servers: + - server: voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/ops + comment: sgm +default_vo_alice: + name: alice + servers: + - server: voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15000' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /not/used +default_vo_generic_server_primary: voms2.cern.ch +default_vo_ops: + name: ops + servers: + - server: voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + - server: lcg-voms2.cern.ch + port: '15009' + dn: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch + ca_dn: /DC=ch/DC=cern/CN=CERN Grid Certification Authority + default_se: not_used.some-domain + sw_dir: . + storage_dir: /storage/ops +default_group_dteam: + gid: 2688 + name: dteam +meta_info_cream-ce: + component: Cream-CE + type: compute_element + version: 1.2 + docker_hub_tag: '' + primary_config_key: cream-info + site_level_config_version: 1.0 + default_var_prefix: cream_default_var + runtime_vars: + - cream_runtime_var_ce_host + host_requirements: + firewall: + - ports: 22 + protocol: tcp + action: accept + ipv6: true + - ports: 8443 + protocol: tcp + action: accept + ipv6: false + - ports: 20000-21000 + protocol: tcp + action: accept + ipv6: false + - ports: 49152 + protocol: tcp + action: accept + ipv6: false + - ports: 49155 + protocol: tcp + action: accept + ipv6: false + - ports: 49152 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9001 + protocol: tcp + action: accept + ipv6: false + - ports: 3306 + protocol: tcp + action: accept + ipv6: false + - ports: 2170 + protocol: tcp + action: accept + ipv6: false + - ports: 33333 + protocol: tcp + action: accept + ipv6: false + - ports: 56565 + protocol: tcp + action: accept + ipv6: false + - ports: 56554 + protocol: tcp + action: accept + ipv6: false + - ports: 40559 + protocol: tcp + action: accept + ipv6: false + cvmfs: true + host_certificates: true + docker_run_parameters: + privileged: true + hostname: lightweight_component01.cern.ch + ports: + - 8443:8443 + - 2811:2811 + - 20000-21000:20000-21000 + - 49152:49152 + - 49155:49155 + - 9002:9002 + - 9001:9001 + - 3306:3306 + - 2170:2170 + - 33333:33333 + - 56565:56565 + - 56554:56554 + - 40559:40559 + tty: true + interactive: true + detached: true + mount: + source: /cvmfs + target: /cvmfs + type: bind + command: /bin/bash +default_vo_generic_dn_server_secondary: /DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch +default_pool_account_ops: + base_name: ops + initial_uid: 45001 + users_num: 10 + primary_group: + gid: 45000 + name: ops + secondary_groups: [] +wn-pbs-expected-from-site-level-config: + pbs_default_var_px_host: myproxy.cern.ch + pbs_default_var_ce_host: lightweight_component01.cern.ch + pbs_default_var_ce_smpsize: 8 + pbs_default_var_batch_server: lightweight_component01.cern.ch + pbs_default_var_ce_close_se: SE3 + pbs_default_var_ce_close_se3_access_point: /not/used + pbs_default_var_ce_close_se3_host: not-used.some-domain + pbs_default_var_se_list: not-used.some-domain + pbs_default_var_se_mount_info_list: none +default_group_alice: + gid: 1395 + name: alice +default_vo_generic_server_secondary: lcg-voms2.cern.ch +default_group_opssgm_name: opssgm +default_vo_generic_dn_server_primary: /DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch +cream-ce-expected-from-site-level-config: + cream_default_var_globus_tcp_port_range: + begin: 20000 + end: 21000 + cream_default_var_ce_capability: CPUScalingReferenceSI00=1000 + cream_default_var_ce_cpu_model: Xeon + cream_default_var_ce_cpu_speed: 2800 + cream_default_var_ce_cpu_vendor: intel + cream_default_var_ce_inboundip: false + cream_default_var_ce_logcpu: 0 + cream_default_var_ce_minphysmem: 2000 + cream_default_var_ce_minvirtmem: 8192 + cream_default_var_ce_os_arch: x86_64 + cream_default_var_ce_otherdescr: Cores=4 + cream_default_var_ce_outboundip: true + cream_default_var_ce_physcpu: 0 + cream_default_var_ce_runtimeenv: EMI-3 + cream_default_var_ce_sf00: 0 + cream_default_var_ce_si00: 381 + cream_default_var_ce_smpsize: 8 + cream_default_var_batch_log_dir: /var/spool/pbs + cream_default_var_blparser_with_updater_notifier: true + cream_default_var_blparser_host: lightweight_component01.cern.ch + cream_default_var_cemon_host: lightweight_component01.cern.ch + cream_default_var_cream_db_user: cream + cream_default_var_apel_mysql_host: lightweight_component01.cern.ch + cream_default_var_ce_close_se: SE3 + cream_default_var_ce_close_se3_access_point: /not/used + cream_default_var_ce_close_se3_host: not-used.some-domain + cream_default_var_se_list: not-used.some-domain + cream_default_var_se_mount_info_list: none +default_group_dteamsgm_name: dteamsgm +default_pool_account_opssgm: + base_name: sgmops + initial_uid: 60701 + users_num: 10 + primary_group: + gid: 45000 + name: ops + secondary_groups: + - gid: 46001 + name: opssgm +default_group_dteam_name: dteam diff --git a/tests/tests_semantics.py b/tests/tests_semantics.py index 9ddb817..4f00d79 100644 --- a/tests/tests_semantics.py +++ b/tests/tests_semantics.py @@ -1,19 +1,17 @@ from compiler.semantics import check_yaml_syntax import unittest +from yamllint.config import YamlLintConfig +from yamllint import linter class MyTest(unittest.TestCase): - def test(self): - input = """ - lightweight_components: - - type: compute_element - repository_url: "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream" - nodes: - - node: lw-site-droplet-0 - container_count: 1 - - node: ec2-18-184-37-92.eu-central-1 - container_count: 2 - preferred_tech_stack: - level_2_configuration: yaim - """ + conf = YamlLintConfig('extends: relaxed') - self.assertEqual(input, input) \ No newline at end of file + def test_check_yaml_syntax(self): + augmented_yaml_file = check_yaml_syntax('../.temp/runtime.yaml') + file = open(augmented_yaml_file) + gen = linter.run(file,self.conf) + errors = list(gen) + self.assertEqual(errors,[]) + +if __name__ == '__main__': + unittest.main() From acfcea5323d8b8f47f01bc9c75c6274e028d4774 Mon Sep 17 00:00:00 2001 From: keval nagda Date: Thu, 21 Mar 2019 13:44:51 +0530 Subject: [PATCH 2/3] Update requirements.txt --- .gitignore | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 66deb2e..a49f915 100644 --- a/.gitignore +++ b/.gitignore @@ -102,4 +102,4 @@ venv.bak/ # mypy .mypy_cache/ -.temp \ No newline at end of file +.temp diff --git a/requirements.txt b/requirements.txt index 7fd1b9e..8074d5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ PyYAML==3.12 ruamel.ordereddict==0.4.13 ruamel.yaml==0.15.71 -argparse==1.4.0 \ No newline at end of file +argparse==1.4.0yamllint==1.15.0 \ No newline at end of file From 3de35c6c9666cac1650494513e03cc119aecc22d Mon Sep 17 00:00:00 2001 From: keval nagda Date: Sat, 23 Mar 2019 15:04:07 +0530 Subject: [PATCH 3/3] Update requirements.txt --- requirements.txt | 3 +- tests/data/runtime.yaml | 503 +++++++++++++++++++++++++++++++++++++++ tests/tests_semantics.py | 4 +- 3 files changed, 507 insertions(+), 3 deletions(-) create mode 100644 tests/data/runtime.yaml diff --git a/requirements.txt b/requirements.txt index 8074d5d..e7d64f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML==3.12 ruamel.ordereddict==0.4.13 ruamel.yaml==0.15.71 -argparse==1.4.0yamllint==1.15.0 \ No newline at end of file +argparse==1.4.0 +yamllint==1.15.0 \ No newline at end of file diff --git a/tests/data/runtime.yaml b/tests/data/runtime.yaml new file mode 100644 index 0000000..2dedc5d --- /dev/null +++ b/tests/data/runtime.yaml @@ -0,0 +1,503 @@ +cream_runtime_variables: + - &cream_runtime_var_ce_host lightweight_component01.cern.ch +pbs_runtime_variables: + - &pbs_runtime_var_ce_host lightweight_component01.cern.ch +meta_info_cream-ce: + component : "Cream-CE" + type: compute_element + version : 1.2 + docker_hub_tag : "" + primary_config_key: "cream-info" + site_level_config_version: 1.0 + default_var_prefix: cream_default_var + runtime_vars: + - cream_runtime_var_ce_host + host_requirements: + firewall: + - ports: 22 + protocol: tcp + action: accept + ipv6: true + - ports: 8443 + protocol: tcp + action: accept + ipv6: false + - ports: 20000-21000 + protocol: tcp + action: accept + ipv6: false + - ports: 49152 + protocol: tcp + action: accept + ipv6: false + - ports: 49155 + protocol: tcp + action: accept + ipv6: false + - ports: 49152 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9002 + protocol: tcp + action: accept + ipv6: false + - ports: 9001 + protocol: tcp + action: accept + ipv6: false + - ports: 3306 + protocol: tcp + action: accept + ipv6: false + - ports: 2170 + protocol: tcp + action: accept + ipv6: false + - ports: 33333 + protocol: tcp + action: accept + ipv6: false + - ports: 56565 + protocol: tcp + action: accept + ipv6: false + - ports: 56554 + protocol: tcp + action: accept + ipv6: false + - ports: 40559 + protocol: tcp + action: accept + ipv6: false + + cvmfs: true + host_certificates: true + + docker_run_parameters: + privileged: true + hostname: *cream_runtime_var_ce_host + ports: + - "8443:8443" + - "2811:2811" + - "20000-21000:20000-21000" + - "49152:49152" + - "49155:49155" + - "9002:9002" + - "9001:9001" + - "3306:3306" + - "2170:2170" + - "33333:33333" + - "56565:56565" + - "56554:56554" + - "40559:40559" + tty: true + interactive: true + detached: true + mount: + source: /cvmfs + target: /cvmfs + type: bind + command: /bin/bash + +meta_info_wn-pbs: + #General Component Desciption + component : "WN-Pbs" + type: worker_node + version : 2.5.13 + docker_hub_url : "" + primary_config_key: "pbs-info" + site_level_config_version: 1.0 + default_var_prefix: "pbs_default_var" + runtime_vars: + - pbs_runtime_var_ce_host + host_requirements: + "firewall": + - ports: 8080 + protocol: tcp + action: accept + ipv6: true + + - ports: 6543 + protocol: tcp + action: accept + ipv6: false + "cvmfs": true + + docker_run_parameters: + "privileged": true + "mount": "wn-pbs" + + + +default_vo_generic_dn_ca_cern: &default_vo_generic_dn_ca_cern '/DC=ch/DC=cern/CN=CERN Grid Certification Authority' +default_vo_generic_dn_server_primary: &default_vo_generic_dn_server_primary '/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch' +default_vo_generic_dn_server_secondary: &default_vo_generic_dn_server_secondary '/DC=ch/DC=cern/OU=computers/CN=lcg-voms2.cern.ch' +default_vo_generic_server_primary: &default_vo_generic_server_primary 'voms2.cern.ch' +default_vo_generic_server_secondary: &default_vo_generic_server_secondary 'lcg-voms2.cern.ch' + +## defaults for supported_virtual_organizations +default_vo_alice: &default_vo_alice + name: alice + servers: + - server: *default_vo_generic_server_primary #voms2.cern.ch + port: '15000' + dn: *default_vo_generic_dn_server_primary #'/DC=ch/DC=cern/OU=computers/CN=voms2.cern.ch' + ca_dn: *default_vo_generic_dn_ca_cern #'/DC=ch/DC=cern/CN=CERN Grid Certification Authority' + - server: *default_vo_generic_server_secondary + port: '15000' + dn: *default_vo_generic_dn_server_secondary + ca_dn: *default_vo_generic_dn_ca_cern + default_se: not_used.some-domain + sw_dir: '.' + storage_dir: "/not/used" + +default_vo_ops: &default_vo_ops + name: ops + servers: + - server: *default_vo_generic_server_primary + port: '15009' + dn: *default_vo_generic_dn_server_primary + ca_dn: *default_vo_generic_dn_ca_cern + - server: *default_vo_generic_server_secondary + port: '15009' + dn: *default_vo_generic_dn_server_secondary + ca_dn: *default_vo_generic_dn_ca_cern + default_se: not_used.some-domain + sw_dir: '.' + storage_dir: "/storage/ops" + +default_vo_dteam: &default_vo_dteam + name: dteam + servers: + - server: *default_vo_generic_server_primary + port: '15001' + dn: *default_vo_generic_dn_server_primary + ca_dn: *default_vo_generic_dn_ca_cern + - server: *default_vo_generic_server_secondary + port: '15001' + dn: *default_vo_generic_dn_server_secondary + ca_dn: *default_vo_generic_dn_ca_cern + default_se: not_used.some-domain + sw_dir: '.' + storage_dir: "/storage/dteam" + + +##default values for group names +default_group_alice_name: &default_group_alice_name alice +default_group_alicesgm_name: &default_group_alicesgm_name alicesgm +default_group_dteam_name: &default_group_dteam_name dteam +default_group_dteamsgm_name: &default_group_dteamsgm_name dteamsgm +default_group_ops_name: &default_group_ops_name ops +default_group_opssgm_name: &default_group_opssgm_name opssgm + +##default values for groups +default_group_alice: &default_group_alice + gid: 1395 + name: *default_group_alice_name + +default_group_alicesgm: &default_group_alicesgm + gid: 1397 + name: *default_group_alicesgm_name + +default_group_dteam: &default_group_dteam + gid: 2688 + name: *default_group_dteam_name + +default_group_dteamsgm: &default_group_dteamsgm + gid: 2680 + name: *default_group_dteamsgm_name + +default_group_ops: &default_group_ops + gid: 45000 + name: *default_group_ops_name + +default_group_opssgm: &default_group_opssgm + gid: 46001 + name: *default_group_opssgm_name + +# default pool accounts +default_pool_accounts_alice: &default_pool_accounts_alice + base_name: 'alice' + initial_uid: 10417 + users_num: 5 + primary_group: *default_group_alice + secondary_groups: [] + +default_pool_accounts_alicesgm: &default_pool_accounts_alicesgm + base_name: 'sgmali' + initial_uid: 60101 + users_num: 10 + primary_group: *default_group_alicesgm + secondary_groups: + - *default_group_alice + +default_pool_accounts_dteam: &default_pool_accounts_dteam + base_name: 'dteam' + initial_uid: 18118 + users_num: 10 + primary_group: *default_group_dteam + secondary_groups: [] + +default_pool_accounts_dteamsgm: &default_pool_accounts_dteamsgm + base_name: 'sgmdtm' + initial_uid: 60501 + users_num: 5 + primary_group: *default_group_dteamsgm + secondary_groups: + - *default_group_dteam + +default_pool_account_ops: &default_pool_accounts_ops + base_name: 'ops' + initial_uid: 45001 + users_num: 10 + primary_group: *default_group_ops + secondary_groups: [] + +default_pool_account_opssgm: &default_pool_accounts_opssgm + base_name: 'sgmops' + initial_uid: 60701 + users_num: 10 + primary_group: *default_group_ops + secondary_groups: + - *default_group_opssgm + + +cream-ce-expected-from-site-level-config: + cream_default_var_globus_tcp_port_range: &cream_default_var_globus_tcp_port_range { + begin: 20000, + end: 21000 + } + cream_default_var_ce_capability: &cream_default_var_ce_capability "CPUScalingReferenceSI00=1000" + cream_default_var_ce_cpu_model: &cream_default_var_ce_cpu_model Xeon + cream_default_var_ce_cpu_speed: &cream_default_var_ce_cpu_speed 2800 + cream_default_var_ce_cpu_vendor: &cream_default_var_ce_cpu_vendor intel + cream_default_var_ce_inboundip: &cream_default_var_ce_inboundip false + cream_default_var_ce_logcpu: &cream_default_var_ce_logcpu 0 + cream_default_var_ce_minphysmem: &cream_default_var_ce_minphysmem 2000 + cream_default_var_ce_minvirtmem: &cream_default_var_ce_minvirtmem 8192 + #cream_default_var_ce_os: &cream_default_var_ce_os ScientificCERNSLC #SET + cream_default_var_ce_os_arch: &cream_default_var_ce_os_arch "x86_64" + #cream_default_var_ce_os_release: &cream_default_var_ce_os_release "6.8" #SET + #cream_default_var_ce_os_version: &cream_default_var_ce_os_version Carbon #SET + cream_default_var_ce_otherdescr: &cream_default_var_ce_otherdescr "Cores=4" + cream_default_var_ce_outboundip: &cream_default_var_ce_outboundip true + cream_default_var_ce_physcpu: &cream_default_var_ce_physcpu 0 + cream_default_var_ce_runtimeenv: &cream_default_var_ce_runtimeenv EMI-3 #SET + cream_default_var_ce_sf00: &cream_default_var_ce_sf00 0 + cream_default_var_ce_si00: &cream_default_var_ce_si00 381 + cream_default_var_ce_smpsize: &cream_default_var_ce_smpsize 8 + #batch_server str() #value needs to be hostname of container + #cream_default_var_job_manager: &cream_default_var_job_manager pbs #SET + #cream_default_var_ce_batch_sys: &cream_default_var_ce_batch_sys torque #SET + cream_default_var_batch_log_dir: &cream_default_var_batch_log_dir "/var/spool/pbs" #SET + #cream_default_var_batch_version: &cream_default_var_batch_version 2.5.13-1cri.9nik #SET + cream_default_var_blparser_with_updater_notifier: &cream_default_var_blparser_with_updater_notifier true + cream_default_var_blparser_host: &cream_default_var_blparser_host + __from__: *cream_runtime_var_ce_host + cream_default_var_cemon_host: &cream_default_var_cemon_host + __from__: *cream_runtime_var_ce_host + cream_default_var_cream_db_user: &cream_default_var_cream_db_user cream + #cream_db_password str() + cream_default_var_apel_mysql_host: &cream_default_var_apel_mysql_host + __from__: *cream_runtime_var_ce_host + #apel_db_password + cream_default_var_ce_close_se: &cream_default_var_ce_close_se SE3 + cream_default_var_ce_close_se3_access_point: &cream_default_var_ce_close_se3_access_point "/not/used" #check + cream_default_var_ce_close_se3_host: &cream_default_var_ce_close_se3_host "not-used.some-domain" #check + cream_default_var_se_list: &cream_default_var_se_list + __from__: *cream_default_var_ce_close_se3_host + cream_default_var_se_mount_info_list: &cream_default_var_se_mount_info_list "none" + + + +wn-pbs-expected-from-site-level-config: + pbs_default_var_px_host: &pbs_default_var_px_host myproxy.cern.ch + pbs_default_var_ce_host: &pbs_default_var_ce_host + __from__: *pbs_runtime_var_ce_host + pbs_default_var_ce_smpsize: &pbs_default_var_ce_smpsize 8 + pbs_default_var_batch_server: &pbs_default_var_batch_server + __from__: *pbs_runtime_var_ce_host + pbs_default_var_ce_close_se: &pbs_default_var_ce_close_se SE3 + pbs_default_var_ce_close_se3_access_point: &pbs_default_var_ce_close_se3_access_point /not/used + pbs_default_var_ce_close_se3_host: &pbs_default_var_ce_close_se3_host not-used.some-domain + pbs_default_var_se_list: &pbs_default_var_se_list + __from__: *pbs_default_var_ce_close_se3_host + pbs_default_var_se_mount_info_list: &pbs_default_var_se_mount_info_list none + +### Variable declaration: +vars: + - &lightweight_component01_ip_address 192.168.0.4 + - &lightweight_component02_ip_address 192.168.0.5 + +site: + name: 'simple-dev-site' + email: 'mayank.sharma@cern.ch' + latitude: 46.3 + longitude: 6.2 + location: CERN, Geneva, Switzerland + description: 'A demo site demonstrating the SIMPLE framework for site setup' + website: 'https://home.cern' + support_website: 'https://groups.google.com/forum/#!forum/wlcg-lightweight-sites' + support_email: 'mayank.sharma@cern.ch' + security_email: 'mayank.sharma@cern.ch' + grid: 'wlcg' #site_other_grid: str(), wlcg, egi,osg + tier: 3 + bdii_host: bdii.cern.ch + use_argus: false + +preferred_tech_stack: + level_1_configuration: puppet + level_2_configuration: puppet + container_orchestration: docker-swarm + container: docker + +site_infrastructure: + - fqdn: lightweight_component01.cern.ch + ip_address: *lightweight_component01_ip_address + - fqdn: lightweight_component02.cern.ch + ip_address: *lightweight_component02_ip_address + +lightweight_components: + - type: compute_element + name: Cream-CE + repository_url: "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_ce_cream" + repository_revision: "master" + execution_id: 0 + lifecycle_hooks: + pre_config: + - /etc/simple_grid/lifecycle/ce_pre_config.sh + pre_init: + - /etc/simple_grid/lifecycle/ce_pre_inst1.sh + post_init: + - /etc/simple_grid/lifecycle/ce_post_inst1.sh + deploy: + - node: lightweight_component01.cern.ch + container_count: 1 + preferred_tech_stack: + level_2_configuration: yaim + config: + mysql_password: fteu5746hg + ce_cpu_model: *cream_default_var_ce_cpu_model + ce_cpu_speed: 3000 + ce_cpu_vendor: Intel + ce_os_arch: str() #move back to node definitions? + ce_physcpu: 0 + blparser_with_updater_notifier: true + cream_db_user: str() + cream_db_password: str() + apel_db_password: str() + queues: + alice: + - voms_fqan: '/alice' + pool_accounts: + - *default_pool_accounts_alice + vo: *default_vo_alice + - voms_fqan: '/alice/ROLE=lcgadmin' + pool_accounts: + - *default_pool_accounts_alice + - *default_pool_accounts_alicesgm + vo: *default_vo_alice + supplemental_config: + some_additional_parameter: some_value + + - name: WN-Pbs + type: worker_node + repository_url: "https://github.com/WLCG-Lightweight-Sites/wlcg_lightweight_site_wn_pbs" + repository_revision: "master" + execution_id: 1 + lifecycle_hooks: + pre_config: + - /etc/simple_grid/lifecycle/wn_pre_config.sh + pre_init: + - /etc/simple_grid/lifecycle/wn_pre_inst1.sh + post_init: + - /etc/simple_grid/lifecycle/wn_post_inst1.sh + deploy: + - node: lightweight_component02.cern.ch + container_count: 2 + preferred_tech_stack: + level_2_configuration: yaim + config: + ce_host: *pbs_runtime_var_ce_host + batch_server: pbs + supplemental_config: + some_additional_parameter: some_value + +supported_virtual_organizations: + - *default_vo_alice + - *default_vo_dteam + - *default_vo_ops + +voms_config: + - voms_fqan: '/alice' + pool_accounts: + - *default_pool_accounts_alice + vo: *default_vo_alice + - voms_fqan: '/alice/ROLE=lcgadmin' + pool_accounts: + - *default_pool_accounts_alice + - *default_pool_accounts_alicesgm + vo: *default_vo_alice + comment: sgm + - voms_fqan: '/dteam' + pool_accounts: + - *default_pool_accounts_dteam + vo: *default_vo_dteam + - voms_fqan: '/dteam/ROLE=lcgadmin' + pool_accounts: + - *default_pool_accounts_dteamsgm + - *default_pool_accounts_dteam + vo: *default_vo_dteam + comment: sgm + - voms_fqan: '/ops' + vo: *default_vo_ops + pool_accounts: + - *default_pool_accounts_ops + - voms_fqan: '/ops/ROLE=lcgadmin' + pool_accounts: + - *default_pool_accounts_opssgm + - *default_pool_accounts_ops + vo: *default_vo_ops + comment: sgm + +# - gid: 1395 +# name: *default_group_alice_name +# accepted_voms_extensions: +# vo: 'alice' # add_vo_name_in_default_config +# group: '' +# role: '' +# category: '' +# - gid: 1397 +# name: *default_group_alicesgm_name +# priority_group: sgm +# accepted_voms_extensions: '/alice/ROLE=lcgadmin' +# - *default_group_dteam +# - *default_group_dteamsgm +# - *default_group_ops +# - <<: *default_group_opssgm +# gid: 2018 + +#pool_accounts: +# - base_name: 'alice' +# initial_uid: '10417' +# users_num: '5' +# primary_group: *default_group_alice_name +# - base_name: 'sgmali' +# initial_uid: '60101' +# users_num: '10' +# primary_group: *default_group_alicesgm_name +# secondary_group: *default_group_alice_name +# - *default_pool_account_dteam +# - *default_pool_account_ops + + + diff --git a/tests/tests_semantics.py b/tests/tests_semantics.py index 4f00d79..6b15ed2 100644 --- a/tests/tests_semantics.py +++ b/tests/tests_semantics.py @@ -3,11 +3,11 @@ from yamllint.config import YamlLintConfig from yamllint import linter -class MyTest(unittest.TestCase): +class TestSemantics(unittest.TestCase): conf = YamlLintConfig('extends: relaxed') def test_check_yaml_syntax(self): - augmented_yaml_file = check_yaml_syntax('../.temp/runtime.yaml') + augmented_yaml_file = check_yaml_syntax('./data/runtime.yaml') file = open(augmented_yaml_file) gen = linter.run(file,self.conf) errors = list(gen)