diff --git a/ansible/deploy-superset.yml b/ansible/deploy-superset.yml index 1cdc555d..617b8091 100644 --- a/ansible/deploy-superset.yml +++ b/ansible/deploy-superset.yml @@ -1,5 +1,5 @@ --- -- import_playbook: ansible-version.yml +- include: ansible-version.yml - hosts: hkgmetadb.infra.ooni.io gather_facts: false # no useful facts there :) @@ -10,4 +10,27 @@ db: metadb name: oosuperset password: '{{ hkgsuperset_postgres_password }}' -... + +- hosts: hkgsuperset.ooni.io + become: false + remote_user: root + gather_facts: true + pre_tasks: + - name: bootstap python + raw: if [ ! -x /usr/bin/python ]; then apt-get update && apt-get -y install python-simplejson python-apt; fi + register: output + changed_when: output.stdout != "" + roles: + - role: adm + adm_passwd: + - "{{ passwd.art }}" + - "{{ passwd.darkk }}" + +- hosts: hkgsuperset.ooni.io + gather_facts: false # already gathered + vars: + letsencrypt_nginx: yes + letsencrypt_domains: "hkgsuperset.ooni.io" + roles: + - letsencrypt + - superset diff --git a/ansible/deploy-website-monitoring.yml b/ansible/deploy-website-monitoring.yml new file mode 100644 index 00000000..1e945bc9 --- /dev/null +++ b/ansible/deploy-website-monitoring.yml @@ -0,0 +1,42 @@ +--- +- hosts: dw.wsm.ooni.io + roles: + - role: adm + adm_passwd: + - "{{ passwd.art }}" + - "{{ passwd.sbs }}" + - "{{ passwd.sarath }}" + +# Manually: +# apt install postgresql postgresql-contrib +# sudo -u postgres psql +# create extension postgres_fdw; +# +# CREATE SERVER metadb FOREIGN DATA WRAPPER +# postgres_fdw OPTIONS +# (dbname 'metadb', host 'ec2-3-92-83-13.compute-1.amazonaws.com', port '5432'); +# +# CREATE USER MAPPING FOR postgres +# SERVER metadb +# OPTIONS (user 'postgres'); +# +# Create the FOREIGN tables using: https://gist.github.com/hellais/6effbf5c728469e67482a60ae90aeb5c +# Move the data directory of postgres into /srv/postgresql_data_dir: +# https://www.dbrnd.com/2018/04/postgresql-move-main-data-directory-in-linux-ubuntu-16-4/ + +- hosts: dw.wsm.ooni.io + gather_facts: false # already gathered + roles: + - role: letsencrypt + letsencrypt_nginx: yes + letsencrypt_domains: ["dw.wsm.ooni.io"] + tags: letsencrypt + - role: superset + letsencrypt_domains: ["dw.wsm.ooni.io"] + superset_web_path: "/superset" + tags: superset + - role: jupyter + jupyter_web_path: "/jupyter" + jupyter_password_hash: "{{ vault_jupyter_password_hash }}" + letsencrypt_domains: ["dw.wsm.ooni.io"] + tags: jupyter diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 62e1f7ca..e09e3571 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -44,6 +44,7 @@ passwd: piwik: {login: piwik, id: 2300, comment: Piwik user} grav: {login: grav, id: 2400, comment: Grav user} + superset: {login: superset, group: superset, id: 2401, comment: Superset User} prtreg: {login: prtreg, id: 4800, comment: Orchestration Registry} prtevent: {login: prtevent, id: 4801, comment: Orchestration Events} diff --git a/ansible/host_vars/dw.wsm.ooni.io/vault b/ansible/host_vars/dw.wsm.ooni.io/vault new file mode 100644 index 00000000..47de6658 --- /dev/null +++ b/ansible/host_vars/dw.wsm.ooni.io/vault @@ -0,0 +1,17 @@ +$ANSIBLE_VAULT;1.1;AES256 +39643036643739646337383131666434363236313062353639393230356538643430613464343138 +3238353430666438393430353466656333626139343632380a393966316566316134363934393237 +35303733666233303165646136383866316430313564373033643031363938326366643063313137 +6335376532303838300a373063383334353361356538316434616563303639353137353533666536 +38396166353935663730663736356166366464366462393863343565666165373932663338363865 +66373833316330393530633963336632613564333236343565323037363064666630636534363765 +35656661343537666435373864636665393039636238636432653961343231393134653239346434 +63623466306462626236306532396430333461323261633032333530623335666539373636663161 +32376262326338613031323939326163656564313965663033343934306165336637323065326130 +32383135623938343433613937353330393236323734333939643833313930653662336161623666 +66333765366131356466613437613735366564353633663731383563646334333062373032353039 +34323938373236323862633938396438376130323662333031613137346435646339656336336130 +33303166646138666139356461373139316337303531393666316132623461326234623133623930 +36356464613564616236363062376135346639656134663265356233633339653863333638353066 +34346663336536353132333239393565383537643865383265336232626362333136616437613431 +61336661633165323263 diff --git a/ansible/host_vars/hkgsuperset.ooni.io/vault b/ansible/host_vars/hkgsuperset.ooni.io/vault new file mode 100644 index 00000000..5f6192bb --- /dev/null +++ b/ansible/host_vars/hkgsuperset.ooni.io/vault @@ -0,0 +1,11 @@ +$ANSIBLE_VAULT;1.1;AES256 +30646230393239663434653430343837336665633132396463326631373436343636353566346363 +3861393335663865623030386366393537333030346136320a303961356563303232313066396666 +31663639333932323764663736383865396339636237353866653036623965316633386530346138 +3436616364386534620a373062353438646561613963316365636364376633353632396566653338 +39343132393364333633316163393836393134616365346662333561373164383037373736643333 +37343862353131643161393932333164616363643330326137383630333938356539383433356661 +34386632636634396564346539306535663565393731616133323166626362616534366564303562 +34313539303632663031373634353930333135383162386235306236643639306639376636346461 +31626436623439323630383330656666313163346364306363613463363135333165366137376437 +3464306138646166376335613465326437376564643139663334 diff --git a/ansible/inventory b/ansible/inventory index 13d8686c..ea330405 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -83,6 +83,9 @@ deb.ooni.nu # NOT-GH, moritz? [bigv] # Bytemark @ London b.echo.th.ooni.io # NOT-GH, down, mirror1.reports.ooni.nu ooni-1.default.orgtech.uk0.bigv.io, OK (u: $login) +[otf-aws] +dw.wsm.ooni.io + ######################################################################## # PSK (pre-shared key) tags diff --git a/ansible/roles/jupyter/defaults/main.yml b/ansible/roles/jupyter/defaults/main.yml index c0af86fa..c3e92750 100644 --- a/ansible/roles/jupyter/defaults/main.yml +++ b/ansible/roles/jupyter/defaults/main.yml @@ -30,4 +30,5 @@ jupyter_user: "{{ passwd.jupyter.login }}" jupyter_group: "{{ passwd.jupyter.login }}" jupyter_port: 8080 +jupyter_web_path: "/" jupyter_password_hash: "{{ CHANGE_ME }}" diff --git a/ansible/roles/jupyter/tasks/setup-anaconda.yml b/ansible/roles/jupyter/tasks/setup-anaconda.yml index 36ca656d..61a36823 100644 --- a/ansible/roles/jupyter/tasks/setup-anaconda.yml +++ b/ansible/roles/jupyter/tasks/setup-anaconda.yml @@ -59,9 +59,3 @@ command: '{{anaconda_link_dir}}/bin/pip install {{ item }}' with_items: "{{ pip_pkgs }}" -- name: remove conda-curl since it conflicts with the system curl - become: yes - become_user: root - command: '{{anaconda_conda_bin}} remove -y curl' - args: - removes: '{{anaconda_link_dir}}/lib/libcurl.a' diff --git a/ansible/roles/jupyter/tasks/setup-jupyter.yml b/ansible/roles/jupyter/tasks/setup-jupyter.yml index 2f958dd8..96e07170 100644 --- a/ansible/roles/jupyter/tasks/setup-jupyter.yml +++ b/ansible/roles/jupyter/tasks/setup-jupyter.yml @@ -1,4 +1,10 @@ --- +- name: ensure juypter user exists + user: + name: "{{ jupyter_user }}" + shell: /bin/sh + state: present + - name: set permissions, owner and group file: path: "{{ jupyter_path }}" diff --git a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 index b943248e..289aa6cf 100644 --- a/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 +++ b/ansible/roles/jupyter/templates/nginx-site-jupyter.j2 @@ -1,14 +1,14 @@ server { server_name _; - listen 443 ssl default_server; - listen [::]:443 ssl default_server; + listen 443 ssl; + listen [::]:443 ssl; - ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem; - ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem; + ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem; + ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem; - location / { + location {{ jupyter_web_path }} { proxy_pass http://127.0.0.1:{{ jupyter_port }}; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; diff --git a/ansible/roles/superset/defaults/main.yml b/ansible/roles/superset/defaults/main.yml new file mode 100644 index 00000000..5a854b00 --- /dev/null +++ b/ansible/roles/superset/defaults/main.yml @@ -0,0 +1,16 @@ +superset_path: /srv/superset +superset_config_path: "{{ superset_path }}/config" +superset_venv_path: "{{ superset_path }}/venv" +superset_port: "8088" +superset_listen_address: "127.0.0.1:{{ superset_port }}" +superset_secret_key: "{{ vault_superset_secret_key }}" +superset_version: "0.28.1" + +superset_admin_username: "admin" +superset_admin_password: "{{ vault_superset_admin_password }}" + +superset_user: "{{ passwd.superset.login }}" +superset_group: "{{ passwd.superset.group }}" +superset_uid: "{{ passwd.superset.id }}" +superset_user_comment: "{{ passwd.superset.comment }}" + diff --git a/ansible/roles/superset/tasks/main.yml b/ansible/roles/superset/tasks/main.yml new file mode 100644 index 00000000..6dca920a --- /dev/null +++ b/ansible/roles/superset/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: create superset group + group: + name: "{{ superset_group }}" + state: present + +- name: create superset user + user: + name: "{{ superset_user }}" + group: "{{ superset_group }}" + uid: "{{ superset_uid }}" + createhome: yes # Superset needs a home + shell: /bin/bash # XXX maybe we can harden this + comment: "{{ superset_user_comment }}" + state: present + +- name: install apt requirements + apt: + name: "{{ item }}" + with_items: + - "build-essential" + - "libssl-dev" + - "libffi-dev" + - "python3-dev" + - "python3-pip" + - "python3-venv" + - "libsasl2-dev" + - "libldap2-dev" + - "libpq-dev" + +- name: mkdir for config and data + file: + path: "{{ item }}" + state: directory + mode: "u=rwx,g=rx,o=" + owner: "{{ superset_user }}" + group: "{{ superset_group }}" + with_items: + - "{{ superset_path }}" + - "{{ superset_config_path }}" + +- name: configure superset + template: + src: superset_config.py.j2 + dest: "{{ superset_config_path }}/superset_config.py" + +- name: create superset virtualenv + pip: + name: "{{ item }}" + virtualenv_command: "/usr/bin/python3 -m venv" + virtualenv: "{{ superset_venv_path }}" + with_items: + - "sqlalchemy==1.2.18" # fixes: https://github.com/apache/incubator-superset/issues/6977 + - "pandas==0.23.4" # Workaround for: https://github.com/apache/incubator-superset/issues/6770 + - "superset=={{ superset_version }}" + - "psycopg2" + become: true + become_user: "{{ superset_user }}" + +- name: create admin user + command: "{{ superset_venv_path}}/bin/fabmanager create-admin --app superset --username {{ superset_admin_username}} --password {{ superset_admin_password }} --firstname admin --lastname admin --email admin@openobservatory.org" + environment: + PYTHONPATH: "{{ superset_config_path }}" # We pass the PYTHONPATH so that superset can get our custom config + become: true + become_user: "{{ superset_user }}" + +- name: upgrade db + command: "{{ superset_venv_path}}/bin/superset db upgrade" + environment: + PYTHONPATH: "{{ superset_config_path }}" + become: true + become_user: "{{ superset_user }}" + +- name: init db + command: "{{ superset_venv_path}}/bin/superset init" + environment: + PYTHONPATH: "{{ superset_config_path }}" + become: true + become_user: "{{ superset_user }}" + +- name: Install superset.service + template: + src: superset.service.j2 + dest: "/etc/systemd/system/superset.service" + +- name: Start superset.service + systemd: + name: superset.service + state: started + enabled: yes + daemon_reload: yes + +- include: setup-nginx.yml diff --git a/ansible/roles/superset/tasks/setup-nginx.yml b/ansible/roles/superset/tasks/setup-nginx.yml new file mode 100644 index 00000000..13b1916e --- /dev/null +++ b/ansible/roles/superset/tasks/setup-nginx.yml @@ -0,0 +1,31 @@ +--- +- name: Install nginx + environment: + RUNLEVEL: 1 # To avoid nginx being launched right after it's installed + apt: + name: "{{ item }}" + state: present + with_items: + - "nginx" + - "python-passlib" + +- name: Remove default nginx virtual host + file: + name: /etc/nginx/sites-enabled/default + state: absent + register: nginx + +- name: Add nginx virtual host config + template: + src: superset_nginx.conf.j2 + dest: /etc/nginx/sites-enabled/superset + owner: root + group: root + mode: 0644 + register: nginx + +- name: Restart nginx + service: + name: nginx + state: restarted + when: nginx.changed diff --git a/ansible/roles/superset/templates/superset.service.j2 b/ansible/roles/superset/templates/superset.service.j2 new file mode 100644 index 00000000..5a8fdab1 --- /dev/null +++ b/ansible/roles/superset/templates/superset.service.j2 @@ -0,0 +1,12 @@ +[Service] +Environment="PYTHONPATH={{ superset_config_path }}" +Environment="PATH={{ superset_venv_path }}/bin:/usr/local/bin:/usr/bin:/bin" +Type=simple +User={{ superset_user }} +Group={{ superset_user }} +ExecStart={{ superset_venv_path }}/bin/superset runserver -p {{ superset_port }} +StandardOutput=syslog +StandardError=syslog + +[Install] +WantedBy=default.target diff --git a/ansible/roles/superset/templates/superset_config.py.j2 b/ansible/roles/superset/templates/superset_config.py.j2 new file mode 100644 index 00000000..113f40da --- /dev/null +++ b/ansible/roles/superset/templates/superset_config.py.j2 @@ -0,0 +1,28 @@ +#--------------------------------------------------------- +# Superset specific config +#--------------------------------------------------------- +ROW_LIMIT = 5000 +SUPERSET_WORKERS = 4 + +SUPERSET_WEBSERVER_PORT = {{ superset_port }} +#--------------------------------------------------------- + +#--------------------------------------------------------- +# Flask App Builder configuration +#--------------------------------------------------------- +# Your App secret key +SECRET_KEY = '{{ superset_secret_key }}' + +# The SQLAlchemy connection string to your database backend +# This connection defines the path to the database that stores your +# superset metadata (slices, connections, tables, dashboards, ...). +# Note that the connection information to connect to the datasources +# you want to explore are managed directly in the web UI +SQLALCHEMY_DATABASE_URI = 'sqlite:////{{ superset_config_path }}/superset.db' + +# Flask-WTF flag for CSRF +WTF_CSRF_ENABLED = True + +# Set this API key to enable Mapbox visualizations +MAPBOX_API_KEY = '' + diff --git a/ansible/roles/superset/templates/superset_nginx.conf.j2 b/ansible/roles/superset/templates/superset_nginx.conf.j2 new file mode 100644 index 00000000..5b43cc64 --- /dev/null +++ b/ansible/roles/superset/templates/superset_nginx.conf.j2 @@ -0,0 +1,26 @@ +# ansible-managed in ooni-sysadmin.git/ansible/roles/superset/templates/superset_nginx.conf.j2 + +{% import 'common.j2' as c %} +server { + server_name _; + + listen 80; + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + + ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/privkey.pem; + ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains[0] }}/fullchain.pem; + + location / { + proxy_pass http://{{ superset_listen_address }}; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + } + + {{ c.location_letsencrypt() }} +} diff --git a/ansible/roles/wiki/tasks/main.yml b/ansible/roles/wiki/tasks/main.yml new file mode 100644 index 00000000..63ddff8d --- /dev/null +++ b/ansible/roles/wiki/tasks/main.yml @@ -0,0 +1,83 @@ +--- + +- name: Install apt-transport-https + apt: + name: apt-transport-https + +- name: Add NodeSource apt package signing key + apt_key: + url: "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" + +- name: Add Mongodb apt package signing key + apt_key: + url: "https://www.mongodb.org/static/pgp/server-3.4.asc" + +- name: Add MongoDB apt repository + apt_repository: + repo: 'deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.4 multiverse' + +- name: Add NodeSource apt repository + apt_repository: + repo: 'deb https://deb.nodesource.com/node_7.x {{ ansible_distribution_release }} main' + +- name: Install required apt packages + apt: + name: "{{ item }}" + with_items: + - git-core + - nodejs + - nginx + - mongodb-org + - bzip2 + +- name: Create wiki user + user: + name: "{{ wiki_user }}" + +- name: Creating wiki dirs + file: + path: "{{ item }}" + state: directory + mode: 0755 + with_items: + - "{{ wiki_user_path }}" + +- name: Remove default nginx virtual host config + file: + name: /etc/nginx/sites-enabled/default + state: absent + register: nginx + +- name: Add wiki nginx virtual host config + template: + src: wiki_nginx.conf.j2 + dest: /etc/nginx/sites-enabled/wiki + owner: root + group: root + mode: 0644 + register: nginx + +- name: Restart nginx + service: + name: nginx + state: restarted + when: nginx.changed + +- name: Install wiki.js package + npm: + name: "{{ item }}" + global: yes + with_items: + - wiki.js + +- name: Install node-wiki.service + template: + src: node-wiki.service.j2 + dest: "{{ node_wiki_unit_path }}" + +- name: Start node-wiki.service + systemd: + name: node-wiki.service + state: started + enabled: yes + daemon_reload: yes diff --git a/ansible/roles/wiki/templates/node-wiki.service.j2 b/ansible/roles/wiki/templates/node-wiki.service.j2 new file mode 100644 index 00000000..dbdbf5b6 --- /dev/null +++ b/ansible/roles/wiki/templates/node-wiki.service.j2 @@ -0,0 +1,11 @@ +[Service] +ExecStart=/usr/bin/node wiki start +Restart=always +StandardOutput=syslog +StandardError=syslog +User={{ wiki_user }} +Group={{ wiki_user }} +RestartSec=15s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/wiki/templates/wiki_nginx.conf.j2 b/ansible/roles/wiki/templates/wiki_nginx.conf.j2 new file mode 100644 index 00000000..eb82e26d --- /dev/null +++ b/ansible/roles/wiki/templates/wiki_nginx.conf.j2 @@ -0,0 +1,19 @@ +server { + server_name _; + + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + + ssl_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/privkey.pem; + ssl_trusted_certificate /etc/letsencrypt/live/{{ letsencrypt_domains.split(',')[0] }}/fullchain.pem; + + location / { + proxy_pass http://127.0.0.1:3000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + } +} diff --git a/ansible/roles/wiki/vars/main.yml b/ansible/roles/wiki/vars/main.yml new file mode 100644 index 00000000..b2a251d9 --- /dev/null +++ b/ansible/roles/wiki/vars/main.yml @@ -0,0 +1,3 @@ +wiki_user: wiki +wiki_user_path: "/home/{{ wiki_user }}/data" +node_wiki_unit_path: /etc/systemd/system/node-wiki.service diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..30a4f943 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible==2.2.2 +boto