Devops toy sba

.gitignore

@@ -4,8 +4,9 @@
 *__pycache__*
 webapp/logs
 webapp/static_data/teachers.yaml
-creds.yaml
-CREDS.YAML
+learning_observer/*/creds.yaml
+learning_observer/*/CREDS.YAML
+devops/tasks/settings/CREDS.YAML
 uncommitted
 extension.crx
 extension.pem

devops/ansible/README.md

@@ -0,0 +1,61 @@
+This is a set of ansible playbooks to setup a new server, currently reliant on the python tasks to assist in the server creation.
+
+To setup a new server, do the following:
+
+  cd ../tasks
+  sudo inv initialize [machine]
+  ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=[machine]"
+  ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit [machine]
+  ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit [machine]
+  ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit [machine]
+  ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit [machine]
+  ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit [machine]
+
+  ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine]
+  inv  inv certbot [machine]
+  ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine]
+
+  ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit [machine]
+
+  TO DO: replace the hosts.ini with a python script that returns inv list output in correct format
+
+  Example for daclassroom:
+
+  sudo inv initialize daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom"
+  <at this point, we can modify the yaml & files in the flock-project/daclassroom/yaml>
+  ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit daclassroom
+  inv certbot daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom
+
+  alternately, we can just use the master playbook to run steps B-F in one go:
+  sudo inv initialize daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom"
+  <at this point, we can modify the yaml & files in the flock-project/daclassroom/yaml>
+  ansible-playbook -i hosts.ini ../ansible/tasks/master_playbook.yaml --limit daclassroom
+  inv certbot daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom
+  ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit daclassroom
+
+  The hosts.ini file can be created by running python hosts.py in the tasks directory.
+  To remove the hosts.ini file completely, you can call the dynamic_hosts.py script. First make sure the script is executable:
+  chmod +x ./dynamic_hosts.py
+
+  Make sure you're logged into aws by running:
+  aws sso login
+
+  Then call a playbook usign the dyanmic_hosts.py script like this:
+
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/B_install_baseline.yaml --limit coglabs
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/C_install_repos.yaml --limit coglabs
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/D_copy_files.yaml --limit coglabs
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/E_setup_os_environment_variables.yaml --limit coglabs
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/F_run_additional_tasks.yaml --limit coglabs
+  inv certbot daclassroom
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/reboot.yaml --limit coglabs
+  ansible-playbook -i dynamic_hosts.py ../ansible/tasks/G_download_config.yaml --limit coglabs
+

devops/ansible/default_flock/files.yaml

@@ -0,0 +1,9 @@
+files:
+  - name: authorized_keys
+  - name: htpasswd
+  - name: nginx
+  - name: creds.yaml
+  - name: lo.sh
+  - name: passwd.lo
+  - name: systemd
+

devops/ansible/default_flock/files/authorized_keys

@@ -0,0 +1,2 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl

devops/ansible/default_flock/files/creds.yaml

@@ -0,0 +1,37 @@
+hostname: {{hostname}}.{{server_domain}}
+auth:
+  password_file: passwd.lo
+pubsub:
+    type: redis
+kvs:
+    # stub for in-memory debugging
+    # redis_ephemeral for redis / debugging (object disappear)
+    # redis for deploy
+    # If using redis_ephemeral, persist objects for 60s = 1 minute
+    # I typically use:
+    # * 1-10s for test cases
+    # * 1-5 minutes for interactive debugging
+    # * 6-24 hours for development
+  default:
+    type: stub
+    expiry: 6000
+  memoization:
+    type: redis_ephemeral
+    expiry: 60
+feature_flags: {}
+roster_data:
+    source: all
+aio:
+    session_secret: {{RANDOM1}}
+    session_max_age: 3600
+config:
+    run_mode: dev
+    debug: []
+theme:
+    server_name: Learning Observer
+    front_page_pitch: Learning Observer is an experimental dashboard. If you'd like to be part of the experiment, please contact us. If you're already part of the experiment, log in!
+    logo_big: /static/media/logo-clean.jpg
+event_auth:
+    local_storage:
+        userfile: students.yaml
+        allow_guest: true

devops/ansible/default_flock/files/htpasswd

@@ -0,0 +1 @@
+coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30

devops/ansible/default_flock/files/nginx

@@ -0,0 +1,40 @@
+server {
+   # We listen for HTTP on port 80. When we set up certbot, this changes to 443.
+   listen 80 default_server;
+   listen [::]:80 default_server;
+
+   server_name {{ hostname }}.{{ server_domain }};
+
+   location / {
+      # Generally, used to configure permissions. E.g. http basic auth, allow/deny
+      # IP blocks, etc. Note that for deploy, this should be broken out into several
+      # blocks (e.g. incoming event, dashboards, etc.)
+      {{nginx_root_options}}
+
+      proxy_pass http://localhost:8888/;
+      proxy_set_header   X-Real-IP    $remote_addr;
+      proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+
+      # We disable CORS globally. This should be more granular.
+      add_header "Access-Control-Allow-Origin"  *;
+      add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD";
+      add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept";
+   }
+   location /wsapi/ {
+      proxy_pass http://localhost:8888/wsapi/;
+      proxy_set_header   X-Real-IP    $remote_addr;
+      proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+      proxy_read_timeout 86400;
+
+      add_header "Access-Control-Allow-Origin"  *;
+      add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD";
+      add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept";
+
+      if ($request_method = OPTIONS ) {
+         return 200;
+      }
+   }
+}

devops/ansible/default_flock/files/systemd

@@ -0,0 +1,14 @@
+[Unit]
+Description=Learning Observer
+
+[Service]
+ExecStart=/home/ubuntu/writing_observer/learning_observer/lo.sh
+Type=simple
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=learning_observer
+User=ubuntu
+Group=ubuntu
+
+[Install]
+WantedBy=multi-user.target

devops/ansible/default_flock/yaml/additional_tasks.yaml

@@ -0,0 +1,59 @@
+- name: Change ownership to ubuntu:ubuntu for /home/ubuntu
+  ansible.builtin.file:
+    path: /home/ubuntu
+    state: directory
+    recurse: yes
+    owner: ubuntu
+    group: ubuntu
+
+- name: Delete the symlink /etc/nginx/sites-enabled/default
+  ansible.builtin.file:
+    path: /etc/nginx/sites-enabled/default
+    state: absent
+
+- name: Delete the symlink /etc/nginx/sites-available/default
+  ansible.builtin.file:
+    path: /etc/nginx/sites-available/default
+    state: absent
+
+- name: Create a symlink for the nginx file
+  ansible.builtin.file:
+    src: /etc/nginx/sites-available/{{ inventory_hostname }}
+    dest: /etc/nginx/sites-enabled/{{ inventory_hostname }}
+    state: link
+
+- name: Ensure virtualenvwrapper is installed
+  ansible.builtin.package:
+    name: virtualenvwrapper
+    state: present
+
+- name: Add virtualenvwrapper initialization to profile
+  ansible.builtin.shell: |
+    echo ". /usr/share/virtualenvwrapper/virtualenvwrapper.sh" >> /home/ubuntu/.profile
+  args:
+    executable: /bin/bash
+
+- name: Source profile and create virtual environment learning_observer
+  ansible.builtin.shell: |
+    cd /home/ubuntu/writing_observer
+    make install
+    . /home/ubuntu/.profile
+    mkvirtualenv learning_observer
+    echo "workon learning_observer" >> /home/ubuntu/.profile
+    . /home/ubuntu/.profile
+    pip install --upgrade pip
+    cd /home/ubuntu/writing_observer/
+    pip install -r requirements.txt
+    cd /home/ubuntu/writing_observer/learning_observer/
+    python setup.py develop
+  args:
+    executable: /bin/bash
+  become: yes
+  become_user: ubuntu
+
+- name: Restart nginx
+  ansible.builtin.service:
+    name: nginx
+    state: restarted
+  become: yes
+

devops/ansible/default_flock/yaml/baseline.yaml

@@ -0,0 +1,73 @@
+    - apt: upgrade=dist update_cache=yes
+
+    - name: Basic utils
+      apt: name={{ item }} 
+      with_items:
+      - curl
+      - emacs
+      - git
+      - git-core
+      - links
+      - lynx
+      - mosh
+      - nmap
+      - whois
+      - screen
+      - wipe
+      - build-essential
+      - net-tools
+
+    # We don't need all of this per se, but it's convenient. If nothing
+    # else, it gives prereqs for `pip`
+    - name: Install Python Packages
+      apt: 
+        name: "{{ item }} "
+        state: present
+      with_items:
+      - ipython3
+      - libxml2-dev
+      - libxslt1-dev
+      - python3-boto
+      - python3-bson
+      - python3-dev
+      - python3-matplotlib
+      - python3-numpy
+      - python3-pandas
+      - python3-pip
+      - python3-scipy
+      - python3-setuptools
+      - python3-sklearn
+      - virtualenvwrapper
+      - libjpeg-dev
+      - python3-opencv
+      - python3-virtualenv
+      - python3-aiohttp
+      - python3-aiohttp-cors
+      - python3-tornado
+      - python3-venv
+      - python3-yaml
+      - python3-asyncpg
+      - python3-bcrypt
+
+    - name: Install server packages
+      apt: 
+        name: "{{ item }}"
+        state: present
+      loop:
+        - redis
+        - nginx
+        - certbot
+        - apache2-utils
+        - fcgiwrap
+        - python3-certbot-nginx
+
+    - name: Install pip
+      apt:
+        name: python3-pip
+        state: present
+
+    - name: Create a virtual environment for AWS CLI
+      command: python3 -m venv /opt/learning-observer
+
+    - name: Install AWS CLI using pip in virtual environment
+      command: /opt/learning-observer/bin/pip install awscli

devops/ansible/default_flock/yaml/files.yaml

@@ -0,0 +1,12 @@
+files:
+  - name: nginx
+    dest: /etc/nginx/sites-available/{hostname}
+  - name: authorized_keys
+    dest: /home/ubuntu/.ssh/authorized_keys
+  - name: passwd.lo
+    dest: /home/ubuntu/writing_observer/learning_observer/passwd.lo
+  - name: lo.sh
+    dest: /home/ubuntu/writing_observer/learning_observer/lo.sh
+  - name: systemd
+    dest: /etc/systemd/system/{hostname}.service
+

devops/ansible/default_flock/yaml/os_env_vars.yaml

@@ -0,0 +1,4 @@
+ os_environment:
+  - key: "ANSIBLE_TEST"
+    value : "test_value"
+

devops/ansible/default_flock/yaml/repos.yaml

@@ -0,0 +1,5 @@
+repos:
+  - name: Writing Observer
+    repo: https://github.com/ETS-Next-Gen/writing_observer.git
+    dest: /home/ubuntu/writing_observer
+    branch: master

devops/ansible/default_flock/yamls.yaml

@@ -0,0 +1,7 @@
+yamls:
+  - name: additional_tasks.yaml
+  - name: baseline.yaml
+  - name: files.yaml
+  - name: repos.yaml
+  - name: os_env_vars.yaml
+

devops/ansible/files_to_upload/authorized_keys

@@ -0,0 +1,2 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl

devops/ansible/files_to_upload/htpasswd

@@ -0,0 +1 @@
+coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30