diff --git a/.gitignore b/.gitignore index d951ebd57..032b07a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,9 @@ *__pycache__* webapp/logs webapp/static_data/teachers.yaml -creds.yaml -CREDS.YAML +learning_observer/*/creds.yaml +learning_observer/*/CREDS.YAML +devops/tasks/settings/CREDS.YAML uncommitted extension.crx extension.pem diff --git a/devops/ansible/README.md b/devops/ansible/README.md new file mode 100644 index 000000000..b212be774 --- /dev/null +++ b/devops/ansible/README.md @@ -0,0 +1,61 @@ +This is a set of ansible playbooks to setup a new server, currently reliant on the python tasks to assist in the server creation. + +To setup a new server, do the following: + + cd ../tasks + sudo inv initialize [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=[machine]" + ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit [machine] + + ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine] + inv inv certbot [machine] + ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine] + + ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit [machine] + + TO DO: replace the hosts.ini with a python script that returns inv list output in correct format + + Example for daclassroom: + + sudo inv initialize daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom" + + ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit daclassroom + inv certbot daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom + + alternately, we can just use the master playbook to run steps B-F in one go: + sudo inv initialize daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom" + + ansible-playbook -i hosts.ini ../ansible/tasks/master_playbook.yaml --limit daclassroom + inv certbot daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit daclassroom + + The hosts.ini file can be created by running python hosts.py in the tasks directory. + To remove the hosts.ini file completely, you can call the dynamic_hosts.py script. First make sure the script is executable: + chmod +x ./dynamic_hosts.py + + Make sure you're logged into aws by running: + aws sso login + + Then call a playbook usign the dyanmic_hosts.py script like this: + + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/B_install_baseline.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/C_install_repos.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/D_copy_files.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/E_setup_os_environment_variables.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/F_run_additional_tasks.yaml --limit coglabs + inv certbot daclassroom + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/reboot.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/G_download_config.yaml --limit coglabs + diff --git a/devops/ansible/default_flock/files.yaml b/devops/ansible/default_flock/files.yaml new file mode 100644 index 000000000..7bc82b71c --- /dev/null +++ b/devops/ansible/default_flock/files.yaml @@ -0,0 +1,9 @@ +files: + - name: authorized_keys + - name: htpasswd + - name: nginx + - name: creds.yaml + - name: lo.sh + - name: passwd.lo + - name: systemd + \ No newline at end of file diff --git a/devops/ansible/default_flock/files/authorized_keys b/devops/ansible/default_flock/files/authorized_keys new file mode 100644 index 000000000..46d2e8ba5 --- /dev/null +++ b/devops/ansible/default_flock/files/authorized_keys @@ -0,0 +1,2 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/ansible/default_flock/files/creds.yaml b/devops/ansible/default_flock/files/creds.yaml new file mode 100644 index 000000000..ce1379199 --- /dev/null +++ b/devops/ansible/default_flock/files/creds.yaml @@ -0,0 +1,37 @@ +hostname: {{hostname}}.{{server_domain}} +auth: + password_file: passwd.lo +pubsub: + type: redis +kvs: + # stub for in-memory debugging + # redis_ephemeral for redis / debugging (object disappear) + # redis for deploy + # If using redis_ephemeral, persist objects for 60s = 1 minute + # I typically use: + # * 1-10s for test cases + # * 1-5 minutes for interactive debugging + # * 6-24 hours for development + default: + type: stub + expiry: 6000 + memoization: + type: redis_ephemeral + expiry: 60 +feature_flags: {} +roster_data: + source: all +aio: + session_secret: {{RANDOM1}} + session_max_age: 3600 +config: + run_mode: dev + debug: [] +theme: + server_name: Learning Observer + front_page_pitch: Learning Observer is an experimental dashboard. If you'd like to be part of the experiment, please contact us. If you're already part of the experiment, log in! + logo_big: /static/media/logo-clean.jpg +event_auth: + local_storage: + userfile: students.yaml + allow_guest: true \ No newline at end of file diff --git a/devops/ansible/default_flock/files/htpasswd b/devops/ansible/default_flock/files/htpasswd new file mode 100644 index 000000000..00db81b64 --- /dev/null +++ b/devops/ansible/default_flock/files/htpasswd @@ -0,0 +1 @@ +coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30 diff --git a/devops/tasks/config/lo.sh b/devops/ansible/default_flock/files/lo.sh similarity index 100% rename from devops/tasks/config/lo.sh rename to devops/ansible/default_flock/files/lo.sh diff --git a/devops/ansible/default_flock/files/nginx b/devops/ansible/default_flock/files/nginx new file mode 100644 index 000000000..3067c6347 --- /dev/null +++ b/devops/ansible/default_flock/files/nginx @@ -0,0 +1,40 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{ hostname }}.{{ server_domain }}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:8888/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + } + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + if ($request_method = OPTIONS ) { + return 200; + } + } +} \ No newline at end of file diff --git a/devops/tasks/config/passwd.lo b/devops/ansible/default_flock/files/passwd.lo similarity index 100% rename from devops/tasks/config/passwd.lo rename to devops/ansible/default_flock/files/passwd.lo diff --git a/devops/ansible/default_flock/files/systemd b/devops/ansible/default_flock/files/systemd new file mode 100644 index 000000000..a789e4513 --- /dev/null +++ b/devops/ansible/default_flock/files/systemd @@ -0,0 +1,14 @@ +[Unit] +Description=Learning Observer + +[Service] +ExecStart=/home/ubuntu/writing_observer/learning_observer/lo.sh +Type=simple +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=learning_observer +User=ubuntu +Group=ubuntu + +[Install] +WantedBy=multi-user.target diff --git a/devops/ansible/default_flock/yaml/additional_tasks.yaml b/devops/ansible/default_flock/yaml/additional_tasks.yaml new file mode 100644 index 000000000..805e9cb11 --- /dev/null +++ b/devops/ansible/default_flock/yaml/additional_tasks.yaml @@ -0,0 +1,59 @@ +- name: Change ownership to ubuntu:ubuntu for /home/ubuntu + ansible.builtin.file: + path: /home/ubuntu + state: directory + recurse: yes + owner: ubuntu + group: ubuntu + +- name: Delete the symlink /etc/nginx/sites-enabled/default + ansible.builtin.file: + path: /etc/nginx/sites-enabled/default + state: absent + +- name: Delete the symlink /etc/nginx/sites-available/default + ansible.builtin.file: + path: /etc/nginx/sites-available/default + state: absent + +- name: Create a symlink for the nginx file + ansible.builtin.file: + src: /etc/nginx/sites-available/{{ inventory_hostname }} + dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} + state: link + +- name: Ensure virtualenvwrapper is installed + ansible.builtin.package: + name: virtualenvwrapper + state: present + +- name: Add virtualenvwrapper initialization to profile + ansible.builtin.shell: | + echo ". /usr/share/virtualenvwrapper/virtualenvwrapper.sh" >> /home/ubuntu/.profile + args: + executable: /bin/bash + +- name: Source profile and create virtual environment learning_observer + ansible.builtin.shell: | + cd /home/ubuntu/writing_observer + make install + . /home/ubuntu/.profile + mkvirtualenv learning_observer + echo "workon learning_observer" >> /home/ubuntu/.profile + . /home/ubuntu/.profile + pip install --upgrade pip + cd /home/ubuntu/writing_observer/ + pip install -r requirements.txt + cd /home/ubuntu/writing_observer/learning_observer/ + python setup.py develop + args: + executable: /bin/bash + become: yes + become_user: ubuntu + +- name: Restart nginx + ansible.builtin.service: + name: nginx + state: restarted + become: yes + \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/baseline.yaml b/devops/ansible/default_flock/yaml/baseline.yaml new file mode 100644 index 000000000..3f3b41758 --- /dev/null +++ b/devops/ansible/default_flock/yaml/baseline.yaml @@ -0,0 +1,73 @@ + - apt: upgrade=dist update_cache=yes + + - name: Basic utils + apt: name={{ item }} + with_items: + - curl + - emacs + - git + - git-core + - links + - lynx + - mosh + - nmap + - whois + - screen + - wipe + - build-essential + - net-tools + + # We don't need all of this per se, but it's convenient. If nothing + # else, it gives prereqs for `pip` + - name: Install Python Packages + apt: + name: "{{ item }} " + state: present + with_items: + - ipython3 + - libxml2-dev + - libxslt1-dev + - python3-boto + - python3-bson + - python3-dev + - python3-matplotlib + - python3-numpy + - python3-pandas + - python3-pip + - python3-scipy + - python3-setuptools + - python3-sklearn + - virtualenvwrapper + - libjpeg-dev + - python3-opencv + - python3-virtualenv + - python3-aiohttp + - python3-aiohttp-cors + - python3-tornado + - python3-venv + - python3-yaml + - python3-asyncpg + - python3-bcrypt + + - name: Install server packages + apt: + name: "{{ item }}" + state: present + loop: + - redis + - nginx + - certbot + - apache2-utils + - fcgiwrap + - python3-certbot-nginx + + - name: Install pip + apt: + name: python3-pip + state: present + + - name: Create a virtual environment for AWS CLI + command: python3 -m venv /opt/learning-observer + + - name: Install AWS CLI using pip in virtual environment + command: /opt/learning-observer/bin/pip install awscli diff --git a/devops/ansible/default_flock/yaml/files.yaml b/devops/ansible/default_flock/yaml/files.yaml new file mode 100644 index 000000000..2cb6d9473 --- /dev/null +++ b/devops/ansible/default_flock/yaml/files.yaml @@ -0,0 +1,12 @@ +files: + - name: nginx + dest: /etc/nginx/sites-available/{hostname} + - name: authorized_keys + dest: /home/ubuntu/.ssh/authorized_keys + - name: passwd.lo + dest: /home/ubuntu/writing_observer/learning_observer/passwd.lo + - name: lo.sh + dest: /home/ubuntu/writing_observer/learning_observer/lo.sh + - name: systemd + dest: /etc/systemd/system/{hostname}.service + \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/os_env_vars.yaml b/devops/ansible/default_flock/yaml/os_env_vars.yaml new file mode 100644 index 000000000..a5c2533db --- /dev/null +++ b/devops/ansible/default_flock/yaml/os_env_vars.yaml @@ -0,0 +1,4 @@ + os_environment: + - key: "ANSIBLE_TEST" + value : "test_value" + \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/repos.yaml b/devops/ansible/default_flock/yaml/repos.yaml new file mode 100644 index 000000000..72792054b --- /dev/null +++ b/devops/ansible/default_flock/yaml/repos.yaml @@ -0,0 +1,5 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: master \ No newline at end of file diff --git a/devops/ansible/default_flock/yamls.yaml b/devops/ansible/default_flock/yamls.yaml new file mode 100644 index 000000000..88456e536 --- /dev/null +++ b/devops/ansible/default_flock/yamls.yaml @@ -0,0 +1,7 @@ +yamls: + - name: additional_tasks.yaml + - name: baseline.yaml + - name: files.yaml + - name: repos.yaml + - name: os_env_vars.yaml + \ No newline at end of file diff --git a/devops/ansible/files_to_upload/authorized_keys b/devops/ansible/files_to_upload/authorized_keys new file mode 100644 index 000000000..46d2e8ba5 --- /dev/null +++ b/devops/ansible/files_to_upload/authorized_keys @@ -0,0 +1,2 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/ansible/files_to_upload/htpasswd b/devops/ansible/files_to_upload/htpasswd new file mode 100644 index 000000000..00db81b64 --- /dev/null +++ b/devops/ansible/files_to_upload/htpasswd @@ -0,0 +1 @@ +coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30 diff --git a/devops/ansible/files_to_upload/nginx b/devops/ansible/files_to_upload/nginx new file mode 100644 index 000000000..3067c6347 --- /dev/null +++ b/devops/ansible/files_to_upload/nginx @@ -0,0 +1,40 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{ hostname }}.{{ server_domain }}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:8888/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + } + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + if ($request_method = OPTIONS ) { + return 200; + } + } +} \ No newline at end of file diff --git a/devops/ansible/files_to_upload/nginx_toy_sba b/devops/ansible/files_to_upload/nginx_toy_sba new file mode 100644 index 000000000..1961236d8 --- /dev/null +++ b/devops/ansible/files_to_upload/nginx_toy_sba @@ -0,0 +1,28 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{hostname}}.{{server_domain}}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:3000/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + auth_basic "Restricted Area"; + auth_basic_user_file /etc/nginx/htpasswd; + + try_files $uri $uri/ =404; + } +} \ No newline at end of file diff --git a/devops/ansible/hosts.ini b/devops/ansible/hosts.ini new file mode 100644 index 000000000..ce16bcb78 --- /dev/null +++ b/devops/ansible/hosts.ini @@ -0,0 +1,3 @@ +[servers] +brostpS2 ansible_host=3.84.57.3 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file diff --git a/devops/ansible/local.yaml b/devops/ansible/local.yaml index 3d502ca93..1ed450fdc 100644 --- a/devops/ansible/local.yaml +++ b/devops/ansible/local.yaml @@ -2,5 +2,4 @@ hosts: localhost connection: local tasks: - - include: tasks/writing-apt.yaml - + - include_tasks: tasks/writing-apt.yaml diff --git a/devops/ansible/tasks/A_setup_flock.yaml b/devops/ansible/tasks/A_setup_flock.yaml new file mode 100644 index 000000000..a3d358e74 --- /dev/null +++ b/devops/ansible/tasks/A_setup_flock.yaml @@ -0,0 +1,72 @@ +- name: Setup flock for target_server if needed + hosts: localhost + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Read YAML files list + include_vars: + file: ../default_flock/yamls.yaml + + - name: Assign variables + set_fact: + dest_dir: "{{ flock_config }}/config/{{ target_server }}/yaml" + default_dir: "../default_flock/yaml" + file_list: "{{ yamls }}" + + - name: Ensure the destination directory exists + file: + path: "{{ dest_dir }}" + state: directory + mode: '0755' + + - name: Check if the file already exists on the server + stat: + path: "{{ dest_dir }}/{{ item.name }}" + register: file_stat + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Copy the file to the server if it does not exist + copy: + src: "{{ default_dir }}/{{ item.name }}" + dest: "{{ dest_dir }}/{{ item.name }}" + when: not (file_stat.results | selectattr('item.name', 'equalto', item.name) | map(attribute='stat.exists') | first) + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Read files list + include_vars: + file: ../default_flock/files.yaml + + - name: Assign variables + set_fact: + dest_dir: "{{ flock_config }}/config/{{ target_server }}/files" + default_dir: "../default_flock/files" + file_list: "{{ files }}" + + - name: Ensure the destination directory exists + file: + path: "{{ dest_dir }}" + state: directory + mode: '0755' + + - name: Check if the file already exists on the server + stat: + path: "{{ dest_dir }}/{{ item.name }}" + register: file_stat + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Copy the file to the server if it does not exist + copy: + src: "{{ default_dir }}/{{ item.name }}" + dest: "{{ dest_dir }}/{{ item.name }}" + when: not (file_stat.results | selectattr('item.name', 'equalto', item.name) | map(attribute='stat.exists') | first) + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" \ No newline at end of file diff --git a/devops/ansible/tasks/B_install_baseline.yaml b/devops/ansible/tasks/B_install_baseline.yaml new file mode 100644 index 000000000..37a34f35b --- /dev/null +++ b/devops/ansible/tasks/B_install_baseline.yaml @@ -0,0 +1,12 @@ +- name: Provision writing analysis server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Include BASELINE.YAML from flock_config + include_tasks: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml/baseline.yaml" diff --git a/devops/ansible/tasks/C_install_repos.yaml b/devops/ansible/tasks/C_install_repos.yaml new file mode 100644 index 000000000..94e904512 --- /dev/null +++ b/devops/ansible/tasks/C_install_repos.yaml @@ -0,0 +1,28 @@ +- name: Install Git Repositories + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "repos.yaml" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + + - name: Clone Git repositories + git: + repo: "https://{{ git_username }}:{{ git_pac }}@{{ item.repo | regex_replace('https://', '') }}" + dest: "{{ item.dest }}" + clone: yes + update: yes + version: "{{ item.branch | default('master') }}" + force: yes + loop: "{{ repos }}" \ No newline at end of file diff --git a/devops/ansible/tasks/D_copy_files.yaml b/devops/ansible/tasks/D_copy_files.yaml new file mode 100644 index 000000000..40c754026 --- /dev/null +++ b/devops/ansible/tasks/D_copy_files.yaml @@ -0,0 +1,49 @@ +- name: Copy files to server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "files.yaml" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + + - name: Ensure destination directories exist + file: + path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" + state: directory + loop: "{{ files }}" + + - name: Generate a random number + set_fact: + random_num: "{{ 1000000 | random }}" + + - name: Generate a random GUID + set_fact: + random_guid: "{{ lookup('pipe', 'uuidgen') }}" + + - name: Copy files to server + template: + src: "{{files_dir }}/{{ item.name }}" + dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + owner: root + group: root + mode: '0644' + loop: "{{ files }}" + vars: + hostname: "{{ inventory_hostname }}" + server_domain: "{{ domain }}" + nginx_root_options: "" + RANDOM1: "{{ random_guid }}" + + \ No newline at end of file diff --git a/devops/ansible/tasks/E_setup_os_environment_variables.yaml b/devops/ansible/tasks/E_setup_os_environment_variables.yaml new file mode 100644 index 000000000..c837846e3 --- /dev/null +++ b/devops/ansible/tasks/E_setup_os_environment_variables.yaml @@ -0,0 +1,26 @@ +- name: Set Environment Variables + hosts: all + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "os_env_vars.yaml" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + + - name: Populate /etc/environment + lineinfile: + path: "/etc/environment" + state: present + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + with_items: "{{ os_environment }}" \ No newline at end of file diff --git a/devops/ansible/tasks/F_run_additional_tasks.yaml b/devops/ansible/tasks/F_run_additional_tasks.yaml new file mode 100644 index 000000000..0c5f5e0d3 --- /dev/null +++ b/devops/ansible/tasks/F_run_additional_tasks.yaml @@ -0,0 +1,17 @@ +- name: Run additional Ansible Tasks + hosts: all + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "additional_tasks.yaml" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + + - name: Include additional tasks + include_tasks: "{{ yaml_dir }}/{{ yaml_file }}" \ No newline at end of file diff --git a/devops/ansible/tasks/G_download_config.yaml b/devops/ansible/tasks/G_download_config.yaml new file mode 100644 index 000000000..c88209111 --- /dev/null +++ b/devops/ansible/tasks/G_download_config.yaml @@ -0,0 +1,34 @@ +- name: Download files from server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "files.yaml" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + + - name: Ensure the local files directory exists + file: + path: "{{ files_dir }}" + state: directory + mode: '0755' + + - name: download files from server + fetch: + src: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + dest: "{{files_dir }}/{{ item.name }}" + flat: yes + loop: "{{ files }}" + + \ No newline at end of file diff --git a/devops/ansible/tasks/master_playbook.yaml b/devops/ansible/tasks/master_playbook.yaml new file mode 100644 index 000000000..fc224c29b --- /dev/null +++ b/devops/ansible/tasks/master_playbook.yaml @@ -0,0 +1,14 @@ +- name: Install baseline packages + import_playbook: B_install_baseline.yaml + +- name: Install repos + import_playbook: C_install_repos.yaml + +- name: Copy files to server + import_playbook: D_copy_files.yaml + +- name: Setup os environment variables + import_playbook: E_setup_os_environment_variables.yaml + +- name: Run additional tasks for server + import_playbook: F_run_additional_tasks.yaml \ No newline at end of file diff --git a/devops/ansible/tasks/reboot.yaml b/devops/ansible/tasks/reboot.yaml new file mode 100644 index 000000000..409439fa4 --- /dev/null +++ b/devops/ansible/tasks/reboot.yaml @@ -0,0 +1,10 @@ +- name: Reboot the server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Reboot the server + reboot: + msg: "Reboot initiated by Ansible" + reboot_timeout: 600 \ No newline at end of file diff --git a/devops/ansible/tasks/writing-apt.yaml b/devops/ansible/tasks/writing-apt.yaml deleted file mode 100644 index e00d6b134..000000000 --- a/devops/ansible/tasks/writing-apt.yaml +++ /dev/null @@ -1,57 +0,0 @@ -- apt: upgrade=dist update_cache=yes - -- name: Basic utils - apt: name={{ item }} - with_items: - - curl - - emacs - - git - - git-core - - links - - lynx - - mosh - - nmap - - whois - - screen - - wipe - - build-essential - - net-tools - -# We don't need all of this per se, but it's convenient. If nothing -# else, it gives prereqs for `pip` -- name: Python - apt: name={{ item }} - with_items: - - ipython3 - - libxml2-dev - - libxslt1-dev - - python3-boto - - python3-bson - - python3-dev - - python3-matplotlib - - python3-numpy - - python3-pandas - - python3-pip - - python3-scipy - - python3-setuptools - - python3-sklearn - - virtualenvwrapper - - libjpeg-dev - - python3-opencv - - python3-virtualenv - - python3-aiohttp - - python3-aiohttp-cors - - python3-tornado - - python3-yaml - - python3-asyncpg - - python3-bcrypt - -- name: Server - apt: name={{ item }} - with_items: - - redis - - nginx - - certbot - - apache2-utils - - fcgiwrap - - python3-certbot-nginx \ No newline at end of file diff --git a/devops/requirements.txt b/devops/requirements.txt index fbb305608..c8fcea507 100644 --- a/devops/requirements.txt +++ b/devops/requirements.txt @@ -2,4 +2,4 @@ chevron boto3 pyyaml fabric - +unzip diff --git a/devops/tasks/README.md b/devops/tasks/README.md index 156977314..91861c76a 100644 --- a/devops/tasks/README.md +++ b/devops/tasks/README.md @@ -89,6 +89,13 @@ inv certbot [machine] inv downloadconfig [machine] ``` +inv provision [machine] can be run step by step as well, using the commands + +sudo inv initialize [machine] +inv baseline [ip address] +inv gitrepos [ip address] +inv venv [ip address] + From there, edit configuration files in `config` and to update the machine to a new version, run @@ -121,4 +128,27 @@ design: atomic commits at the end (e.g. download to a temporary dir, and move right before the commit. * However, it is possible to reverse-engineered exactly what happened, - roughly when. This is good enough for now. \ No newline at end of file + roughly when. This is good enough for now. + + + @@@@PB + setup hosts.ini file, and run the following command to configure the remote server: + (from inside devops/tasks folder) + + sudo inv initialize [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/baseline.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/install_repos.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/copy_files.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit [machine] + + ###If setting up + ansible-playbook -i hosts.ini ../ansible/tasks/toy-sba.yaml --limit [machine] + + inv certbot [machine] + inv downloadconfig [machine] + + +###TODO- replace hosts.ini with a python script that returns inv list output in correct format + + + ansible-playbook -i hosts.ini ../ansible/tasks/baseline.yaml --limit coglabs \ No newline at end of file diff --git a/devops/tasks/config/sync.csv b/devops/tasks/config/sync.csv deleted file mode 100644 index e3592df21..000000000 --- a/devops/tasks/config/sync.csv +++ /dev/null @@ -1,6 +0,0 @@ -creds.yaml,root:root,644,/home/ubuntu/writing_observer/learning_observer/creds.yaml,"Learning Observer settings file" -nginx,root:root,644,/etc/nginx/sites-enabled/{hostname},"nginx site configuration" -passwd.lo,root:root,644,/home/ubuntu/writing_observer/learning_observer/passwd.lo,"(Generally blank) passwords file" -lo.sh,ubuntu:ubuntu,744,/home/ubuntu/writing_observer/learning_observer/lo.sh,"Script to start Learning Observer with a nice process name" -systemd,root:root,644,/etc/systemd/system/learning_observer.service,"Systemd init script" -rsyslog.conf,root:root,644,/etc/rsyslog.d/learning_observer.conf,"rsyslog script (for stdout/stderr)" \ No newline at end of file diff --git a/devops/tasks/config_NO_LONGER_NEEDED/authorized_keys b/devops/tasks/config_NO_LONGER_NEEDED/authorized_keys new file mode 100644 index 000000000..b25f6c109 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/authorized_keys @@ -0,0 +1,4 @@ +Copy the following into: ~/.ssh/authorized_keys + +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/tasks/config/creds.yaml b/devops/tasks/config_NO_LONGER_NEEDED/creds.yaml similarity index 69% rename from devops/tasks/config/creds.yaml rename to devops/tasks/config_NO_LONGER_NEEDED/creds.yaml index dab0fe6d8..b626b36de 100644 --- a/devops/tasks/config/creds.yaml +++ b/devops/tasks/config_NO_LONGER_NEEDED/creds.yaml @@ -1,14 +1,4 @@ hostname: {{hostname}}.{{domain}} -xmpp: - sink: # Receives messages. We'll need many of these. - jid: sink@localhost - password: {{RANDOM1}} - source: # Sends messages. - jid: source@localhost - password: {{RANDOM1}} - stream: # For debugging - jid: stream@localhost - password: {{RANDOM1}} auth: password_file: passwd.lo pubsub: diff --git a/devops/tasks/config/hostname b/devops/tasks/config_NO_LONGER_NEEDED/hostname similarity index 100% rename from devops/tasks/config/hostname rename to devops/tasks/config_NO_LONGER_NEEDED/hostname diff --git a/devops/tasks/config/init.d b/devops/tasks/config_NO_LONGER_NEEDED/init.d similarity index 100% rename from devops/tasks/config/init.d rename to devops/tasks/config_NO_LONGER_NEEDED/init.d diff --git a/devops/tasks/config_NO_LONGER_NEEDED/lo.sh b/devops/tasks/config_NO_LONGER_NEEDED/lo.sh new file mode 100644 index 000000000..d9a05d850 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/lo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# This is a script to start up Learning Observer with it's own process +# name. This is convenient for being able to start / stop the process. + +. /usr/share/virtualenvwrapper/virtualenvwrapper.sh +workon learning_observer +cd /home/ubuntu/writing_observer/learning_observer +bash -c "exec -a learning_observer python learning_observer" >> /home/ubuntu/lo.log 2>> /home/ubuntu/lo.err diff --git a/devops/tasks/config/nginx b/devops/tasks/config_NO_LONGER_NEEDED/nginx similarity index 100% rename from devops/tasks/config/nginx rename to devops/tasks/config_NO_LONGER_NEEDED/nginx diff --git a/devops/tasks/config_NO_LONGER_NEEDED/nginx_toy_sba b/devops/tasks/config_NO_LONGER_NEEDED/nginx_toy_sba new file mode 100644 index 000000000..389af3210 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/nginx_toy_sba @@ -0,0 +1,27 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{hostname}}.{{domain}}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:3000/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + # Enable HTTP Basic Authentication + auth_basic "Restricted Content"; + auth_basic_user_file /etc/nginx/.htpasswd; + } +} \ No newline at end of file diff --git a/devops/tasks/config_NO_LONGER_NEEDED/passwd.lo b/devops/tasks/config_NO_LONGER_NEEDED/passwd.lo new file mode 100644 index 000000000..e69de29bb diff --git a/devops/tasks/config/postuploads b/devops/tasks/config_NO_LONGER_NEEDED/postuploads similarity index 100% rename from devops/tasks/config/postuploads rename to devops/tasks/config_NO_LONGER_NEEDED/postuploads diff --git a/devops/tasks/config/rsyslog.conf b/devops/tasks/config_NO_LONGER_NEEDED/rsyslog.conf similarity index 100% rename from devops/tasks/config/rsyslog.conf rename to devops/tasks/config_NO_LONGER_NEEDED/rsyslog.conf diff --git a/devops/tasks/config_NO_LONGER_NEEDED/sync.csv b/devops/tasks/config_NO_LONGER_NEEDED/sync.csv new file mode 100644 index 000000000..4c294b12a --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/sync.csv @@ -0,0 +1,7 @@ +creds.yaml,root:root,644,/home/ubuntu/writing_observer/learning_observer/creds.yaml,"Learning Observer settings file" +nginx_toy_sba,root:root,644,/etc/nginx/sites-available/{hostname},"nginx site configuration" +authorized_keys,root:root,644,/home/ubuntu/authorized_keys,"nginx site configuration" +passwd.lo,root:root,644,/home/ubuntu/writing_observer/learning_observer/passwd.lo,"(Generally blank) passwords file" +toy-sba.sh,ubuntu:ubuntu,744,/home/ubuntu/toy-sba/toy-sba.sh,"Script to start toy-sba" +systemd_toy_sba,root:root,644,/etc/systemd/system/toy_sba.service,"Systemd init script for toy-sba" +rsyslog.conf,root:root,644,/etc/rsyslog.d/learning_observer.conf,"rsyslog script (for stdout/stderr)" \ No newline at end of file diff --git a/devops/tasks/config/systemd b/devops/tasks/config_NO_LONGER_NEEDED/systemd similarity index 100% rename from devops/tasks/config/systemd rename to devops/tasks/config_NO_LONGER_NEEDED/systemd diff --git a/devops/tasks/config_NO_LONGER_NEEDED/systemd_toy_sba b/devops/tasks/config_NO_LONGER_NEEDED/systemd_toy_sba new file mode 100644 index 000000000..b1e842676 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/systemd_toy_sba @@ -0,0 +1,11 @@ +[Unit] +Description=toy-sba + +[Service] +ExecStart=/home/ubuntu/toy-sba/toy-sba.sh +Type=simple +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=toy_sba +User=ubuntu +Group=ubuntu \ No newline at end of file diff --git a/devops/tasks/config_NO_LONGER_NEEDED/toy-sba.sh b/devops/tasks/config_NO_LONGER_NEEDED/toy-sba.sh new file mode 100644 index 000000000..d3c018ba0 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/toy-sba.sh @@ -0,0 +1,7 @@ +#!/usr/bin/bash + +# This is a script to start up toy-sba with it's own process +# name. This is convenient for being able to start / stop the process. + +cd /home/ubuntu/toy-sba +bash -c "sudo -E npm run dev" >> /home/ubuntu/toy_sba.log 2>> /home/ubuntu/toy_sba.err diff --git a/devops/tasks/dynamic_hosts.py b/devops/tasks/dynamic_hosts.py new file mode 100644 index 000000000..f1beca03f --- /dev/null +++ b/devops/tasks/dynamic_hosts.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json +import orchlib.aws +import orchlib.config + +def get_inventory(): + # Fetch list of instances + instances = orchlib.aws.list_instances() + + # Initialize inventory structure + inventory = { + 'servers': { + 'hosts': [], + 'vars': { + 'ansible_user': 'ubuntu', + 'ansible_ssh_private_key_file': orchlib.config.creds['key_filename'] + } + }, + '_meta': { + 'hostvars': {} + } + } + + # Iterate through the instances and add them to the inventory + for instance in instances: + instance_name = instance['Tags']['Name'] + ip_address = instance['PublicIpAddress'] + + # Add instance to the aws_servers group + inventory['servers']['hosts'].append(instance_name) + + # Add host-specific variables + inventory['_meta']['hostvars'][instance_name] = { + 'ansible_host': ip_address + } + + return inventory + +if __name__ == '__main__': + inventory = get_inventory() + print(json.dumps(inventory, indent=2)) \ No newline at end of file diff --git a/devops/tasks/hosts.ini b/devops/tasks/hosts.ini new file mode 100644 index 000000000..081c06e7c --- /dev/null +++ b/devops/tasks/hosts.ini @@ -0,0 +1,6 @@ +[servers] +open-house ansible_host=3.87.42.157 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +da-i18n ansible_host=54.166.118.68 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +da ansible_host=44.202.219.209 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +daclassroom ansible_host=18.207.219.42 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem diff --git a/devops/tasks/hosts.py b/devops/tasks/hosts.py new file mode 100644 index 000000000..1637ad2ac --- /dev/null +++ b/devops/tasks/hosts.py @@ -0,0 +1,30 @@ +import orchlib.aws +import orchlib.config +import orchlib.fabric_flock +import orchlib.templates +import orchlib.ubuntu +import orchlib.repos +from orchlib.logger import system + +def update_hosts_ini(): + # Fetch list of instances + instances = orchlib.aws.list_instances() + ###print(f"Debug: instances contains: {instances}") + + # Open or create the hosts.ini file + with open('hosts.ini', 'w') as hosts_file: + # Write a default header or group for the ansible inventory + hosts_file.write('[servers]\n') + + key_file = orchlib.config.creds['key_filename'] + + # Iterate through the instances and write their details + for instance in instances: + ## print(f"Debug: instance contains: {instance}") + + instance_name = instance['Tags']['Name'] + ip_address = instance['PublicIpAddress'] + hosts_file.write(f"{instance_name} ansible_host={ip_address} ansible_user=ubuntu ansible_ssh_private_key_file={key_file}\n") + +# Call the function to update the hosts.ini file +update_hosts_ini() \ No newline at end of file diff --git a/devops/tasks/orchlib/aws.py b/devops/tasks/orchlib/aws.py index ce76b14ba..b37f76cae 100644 --- a/devops/tasks/orchlib/aws.py +++ b/devops/tasks/orchlib/aws.py @@ -20,6 +20,7 @@ r53 = boto3.client('route53') UBUNTU_20_04 = "ami-09e67e426f25ce0d7" +UBUNTU_24_04 = "ami-0e86e20dae9224db8" def create_instance(name): ''' @@ -27,11 +28,11 @@ def create_instance(name): ''' blockDeviceMappings = [ { - "DeviceName": "/dev/xvda", + "DeviceName": "/dev/sda1", ###xvda", "Ebs": { "DeleteOnTermination": True, - "VolumeSize": 32, - "VolumeType": "gp2" + "VolumeSize": 16, + "VolumeType": "gp3" } } ] @@ -47,8 +48,12 @@ def create_instance(name): 'Value': orchlib.config.creds['owner'] }, { - 'Key': 'deploy-group', - 'Value': orchlib.config.creds['deploy-group'] + 'Key': 'deploy_group', + 'Value': orchlib.config.creds['deploy_group'] + }, + { + 'Key': 'Patch Group', + 'Value': 'pet' } ] @@ -69,8 +74,8 @@ def create_instance(name): # It doesn't correspond 1:1, but it's a good starting # point. response = ec2.create_instances( - ImageId=UBUNTU_20_04, - InstanceType='t2.small', + ImageId=UBUNTU_24_04, + InstanceType='t3.small', BlockDeviceMappings=blockDeviceMappings, KeyName=orchlib.config.creds['aws_keyname'], MinCount=1, @@ -122,8 +127,8 @@ def list_instances(): ''' reservations = ec2client.describe_instances(Filters=[ { - 'Name': 'tag:deploy-group', - 'Values': [orchlib.config.creds['deploy-group']] + 'Name': 'tag:deploy_group', + 'Values': [orchlib.config.creds['deploy_group']] }, ])['Reservations'] instances = sum([i['Instances'] for i in reservations], []) diff --git a/devops/tasks/orchlib/config.py b/devops/tasks/orchlib/config.py index 330de195c..dcc3205cf 100644 --- a/devops/tasks/orchlib/config.py +++ b/devops/tasks/orchlib/config.py @@ -17,9 +17,14 @@ "aws_security_group": "AWS security group (e.g. sg-012345abc)", "owner": "Your name", "email": "Your email", + "git_username": "Your git username", + "git_pac": "Your git personal access token; See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens", + "openai_url": " URL for OPENAI API", + "openai_deployment_id": "OPEN AI deployment ID", + "openai_api_key": "your api key for the azure instance", "domain": "Domain name (e.g. learning-observer.org)", - "flock-config": "Path to git repo where we'll store machine config.", - "deploy-group": "Tag to identify all machines (typically, learning-observer)", + "flock_config": "Path to git repo where we'll store machine config.", + "deploy_group": "Tag to identify all machines (typically, learning-observer)", "ec2_tags": "JSON dictionary of any additional tags you'd like on your machines. If you're not sure, type {}" } print("I'll need:") @@ -31,9 +36,9 @@ print(value) d[key] = input("{key}: ".format(key=key)).strip() d['ec2_tags'] = json.loads(d['ec2_tags']) - if not os.path.exists(d['flock-config']): - os.system("git init {path}".format(path=d['flock-config'])) - os.mkdir(os.path.join(d['flock-config'], "config")) + if not os.path.exists(d['flock_config']): + os.system("git init {path}".format(path=d['flock_config'])) + os.mkdir(os.path.join(d['flock_config'], "config")) with open("settings/CREDS.YAML", "w") as fp: yaml.dump(d, fp) @@ -54,15 +59,15 @@ def config_filename(machine_name, file_suffix, create=False): paths = [ # First, we try per-machine configuration os.path.join( - creds["flock-config"], "config", machine_name, file_suffix + creds["flock_config"], "config", machine_name, file_suffix ), # Next, we try the per-machine override os.path.join( - creds["flock-config"], "config", machine_name, file_suffix+".base" + creds["flock_config"], "config", machine_name, file_suffix+".base" ), # Then, system-wide configuration os.path.join( - creds["flock-config"], "config", file_suffix + creds["flock_config"], "config", file_suffix ), # And finally, as a fallback, default files os.path.join( diff --git a/devops/tasks/orchlib/logger.py b/devops/tasks/orchlib/logger.py index 2622b4d62..667054121 100644 --- a/devops/tasks/orchlib/logger.py +++ b/devops/tasks/orchlib/logger.py @@ -37,5 +37,5 @@ def exitlog(): Not done. ''' os.path.join( - orchlib.config.creds["flock-config"], "logs" + orchlib.config.creds["flock_config"], "logs" ) diff --git a/devops/tasks/orchlib/ubuntu.py b/devops/tasks/orchlib/ubuntu.py index af4327d40..415270d3d 100644 --- a/devops/tasks/orchlib/ubuntu.py +++ b/devops/tasks/orchlib/ubuntu.py @@ -7,27 +7,89 @@ import orchlib.fabric_flock import orchlib.config - +import tempfile +import os def run_script(scriptfile): ''' Helper which executes a series of commands on set of machines ''' - script = open("scripts/{fn}.fab".format(fn=scriptfile)).read() + print("------------------------------------------") + + # Read the script file + script_path = "scripts/{fn}.fab".format(fn=scriptfile) + print("Running script: ", script_path) + try: + with open(script_path, 'r') as file: + script = file.read() + except FileNotFoundError: + print(f"Script file not found: {script_path}") + return + except Exception as e: + print(f"Error reading script file: {e}") + return + def run(*machines): group = orchlib.fabric_flock.machine_group(*machines) - + i = 0 for line in ['hostname'] + script.split("\n"): line = line.strip() + i = i + 1 if len(line) > 0 and line[0] != "#": - print(line) group.run(line) + print("------------------------------------------") return run update = run_script("update") baseline_packages = run_script("baseline_packages") python_venv = run_script("python_venv") +def install_git_repos(ip): + + git_username = orchlib.config.creds.get('git_username') + git_pac = orchlib.config.creds.get('git_pac') + openai_url = orchlib.config.creds.get('openai_url') + openai_deployment_id = orchlib.config.creds.get('openai_deployment_id') + openai_api_key = orchlib.config.creds.get('openai_api_key') + + if not git_username or not git_pac or not openai_url or not openai_deployment_id or not openai_api_key: + raise ValueError("Git/OpenAI credentials are not available in the config") + + # Path to the baseline_packages.fab file + fab_file_path = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'git_repos.fab') + + # Read the content of the baseline_packages.fab file + with open(fab_file_path, 'r') as file: + content = file.read() + + # Replace placeholders with actual credentials + content = content.replace('{git_username}', git_username) + content = content.replace('{git_pac}', git_pac) + content = content.replace('{openai_url}', openai_url) + content = content.replace('{openai_deployment_id}', openai_deployment_id) + content = content.replace('{openai_api_key}', openai_api_key) + + # Write the updated content to a temporary file + temp_dir = os.path.join(os.path.dirname(__file__), '..', 'scripts') + with tempfile.NamedTemporaryFile(delete=False, suffix='.fab', dir=temp_dir) as temp_file: + temp_file.write(content.encode()) + temp_file_path = temp_file.name + + print(f"Running: {temp_file_path}") + temp_file_name = os.path.basename(temp_file_path) + temp_file_name = temp_file_name.replace('.fab', '') + + # Run the resulting script using run_script + print("Running baseline_packages script: " + temp_file_name) + + try: + run_script(temp_file_name)(ip) + except Exception as e: + print(f"An error occurred while running the script: {e}") + return None + finally: + # Clean up the temporary file + os.remove(temp_file_path) def reboot(machine): ''' @@ -47,7 +109,9 @@ def provision(ip): connect_kwargs={"key_filename": orchlib.config.creds['key_filename']} ) update() - baseline_packages() + + run_baseline_packages() + ###baseline_packages() python_venv() reboot() diff --git a/devops/tasks/scripts/baseline_packages.fab b/devops/tasks/scripts/baseline_packages.fab index fbd88ece4..c6e2d7837 100644 --- a/devops/tasks/scripts/baseline_packages.fab +++ b/devops/tasks/scripts/baseline_packages.fab @@ -1,4 +1,32 @@ + cd -sudo apt-get -y install git ansible awscli -git clone https://github.com/ETS-Next-Gen/writing_observer.git + +sudo apt update +##sudo apt-get install -y virtualenvwrapper +##sudo apt-get -y install nginx +##sudo apt-get install -y certbot python3-certbot-nginx +##sudo nginx -s reload +##sudo apt-get -y install git ansible python3-pip python3-venv pipx + +#sudo apt-get -y install git ansible awscli +sudo apt-get -y install git ansible + +# Check if the writing_observer directory exists before cloning +[ ! -d "writing_observer" ] && git clone https://github.com/ETS-Next-Gen/writing_observer.git || echo "Directory writing_observer already exists. Skipping clone." + +# Navigate to the writing_observer repository directory +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch devops_toy_sba + +## PIOTR: what is the next statement for? It fails with the error: +## ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks. cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml + +# Ensure pipx binaries are available in the PATH +#python3 -m pipx ensurepath + +# Install awscli using pipx +#pipx install awscli + +# Navigate to the writing_observer repository directory +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch pmitros/lo_event + diff --git a/devops/tasks/scripts/git_repos.fab b/devops/tasks/scripts/git_repos.fab new file mode 100644 index 000000000..42785b89c --- /dev/null +++ b/devops/tasks/scripts/git_repos.fab @@ -0,0 +1,34 @@ +#Install npm for the toy-sba project +sudo apt-get -y install npm + +cd + +# Set system variables +export OPENAI_URL="{openai_url}" +export OPENAI_DEPLOYMENT_ID="{openai_deployment_id}" +export OPENAI_API_KEY="{openai_api_key}" + +# Check if the toy-sba directory exists before cloning +[ ! -d "toy-sba" ] && git clone https://{git_username}:{git_pac}@github.com/ETS-Next-Gen/toy-sba.git || echo "Directory toy-sba already exists. Skipping clone." + + +#switch branches for writing_observer and toy-sba +# Navigate to the writing_observer repository directory + +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch pmitros/lo_event + +cd /home/ubuntu/toy-sba && git fetch && git switch writing_survey + +# Navigate to the lo_event module and install npm packages +cd /home/ubuntu/writing_observer/modules/lo_event && npm install + +# Install npm packages for toy-sba +cd /home/ubuntu/toy-sba +rm -rf node_modules package-lock.json +npm install --legacy-peer-deps +npm install react-diff-components --legacy-peer-deps +npm install @fortawesome/fontawesome-svg-core --legacy-peer-deps + + + + diff --git a/devops/tasks/scripts/python_venv.fab b/devops/tasks/scripts/python_venv.fab index 629c773eb..08b5eee5d 100644 --- a/devops/tasks/scripts/python_venv.fab +++ b/devops/tasks/scripts/python_venv.fab @@ -1,6 +1,19 @@ +sudo apt-get install -y virtualenvwrapper + +# Source virtualenvwrapper.sh +export WORKON_HOME=$HOME/.virtualenvs +source /usr/share/virtualenvwrapper/virtualenvwrapper.sh + +source ~/.bashrc +mkvirtualenv --version + +# Create a virtual environment +mkvirtualenv learning_observer + cd echo . /usr/share/virtualenvwrapper/virtualenvwrapper.sh >> ~/.profile -source ~/.profile; mkvirtualenv learning_observer +source ~/.profile; + echo workon learning_observer >> ~/.profile source ~/.profile; pip install --upgrade pip source ~/.profile; cd writing_observer/ ; pip install -r requirements.txt diff --git a/devops/tasks/settings/LO_FILES.YAML b/devops/tasks/settings/LO_FILES.YAML new file mode 100644 index 000000000..0d6697ec1 --- /dev/null +++ b/devops/tasks/settings/LO_FILES.YAML @@ -0,0 +1,6 @@ +files: + - src: ../../ansible/files_to_upload/nginx + dest: /etc/nginx/sites-available/{hostname} + - src: ../../ansible/files_to_upload/authorized_keys + dest: /home/ubuntu/authorized_keys + \ No newline at end of file diff --git a/devops/tasks/settings/TOY_REPOS.YAML b/devops/tasks/settings/TOY_REPOS.YAML new file mode 100644 index 000000000..9d7784e76 --- /dev/null +++ b/devops/tasks/settings/TOY_REPOS.YAML @@ -0,0 +1,9 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: pmitros/lo_event + - name: Toy-SBA + repo: https://github.com/ETS-Next-Gen/toy-sba.git + dest: /home/ubuntu/toy-sba + branch: writing_survey \ No newline at end of file diff --git a/devops/tasks/settings/TOY_SBA_FILES.YAML b/devops/tasks/settings/TOY_SBA_FILES.YAML new file mode 100644 index 000000000..9c87b1548 --- /dev/null +++ b/devops/tasks/settings/TOY_SBA_FILES.YAML @@ -0,0 +1,6 @@ +files: + - src: ../../ansible/files_to_upload/nginx_toy_sba + dest: /etc/nginx/sites-available/{hostname} + - src: ../../ansible/files_to_upload/htpasswd + dest: /etc/nginx/htpasswd + \ No newline at end of file diff --git a/devops/tasks/settings/WO_REPOS.YAML b/devops/tasks/settings/WO_REPOS.YAML new file mode 100644 index 000000000..72792054b --- /dev/null +++ b/devops/tasks/settings/WO_REPOS.YAML @@ -0,0 +1,5 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: master \ No newline at end of file diff --git a/devops/tasks/tasks.py b/devops/tasks/tasks.py index 9339e993d..069dd6f29 100644 --- a/devops/tasks/tasks.py +++ b/devops/tasks/tasks.py @@ -52,6 +52,45 @@ def provision(c, machine_name): orchlib.ubuntu.update(ip) print("Baseline...") orchlib.ubuntu.baseline_packages(ip) + print("Git Repos...") + orchlib.ubuntu.install_git_repos(ip) + print("Venv...") + orchlib.ubuntu.python_venv(ip) + +@task +def initialize(c, machine_name): + ''' + Set up a baseline image with all the packages needed for + Learning Observer. Note that this will **not** configure + the machine. + ''' + print("Provisioning...") + machine_info = orchlib.aws.create_instance(machine_name) + print("Updating...") + ip = machine_info.public_ip_address + print("DNS....") + orchlib.aws.register_dns(machine_name, orchlib.config.creds['domain'], ip) + print("IP", ip) + + # Write to hosts.ini + hosts_ini_path = '../settings/hosts.ini' + line_to_write = f"{machine_name} ansible_host={ip} ansible_user=ubuntu ansible_ssh_private_key_file={orchlib.config.creds['key_filename']}\n" + with open(hosts_ini_path, 'a') as hosts_file: + hosts_file.write(line_to_write) + print(f"Added {machine_name} to {hosts_ini_path}") + +@task +def baseline(c, ip): + print("Baseline...") + orchlib.ubuntu.baseline_packages(ip) + +@task +def gitrepos(c, ip): + print("Git Repos...") + orchlib.ubuntu.install_git_repos(ip) + +@task +def venv(c, ip): print("Venv...") orchlib.ubuntu.python_venv(ip) @@ -327,7 +366,7 @@ def commit(c, msg): ''' system( "cd {gitpath} ; git add -A; git commit -m {msg}".format( - gitpath=orchlib.config.creds["flock-config"], + gitpath=orchlib.config.creds["flock_config"], msg=msg ) ) diff --git a/package.json b/package.json index b5d78de21..467d36f3c 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,8 @@ "stylelint": "^15.5.0", "stylelint-config-standard": "^33.0.0", "stylelint-scss": "^4.6.0" + }, + "dependencies": { + "nodemon": "^3.1.5" } }