From 4846dde681cddb820f4be5def71ba44ab81edc1b Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Thu, 12 Sep 2024 10:54:02 -0400 Subject: [PATCH 01/16] not all scripts run remotely as expected --- devops/requirements.txt | 2 +- devops/tasks/config/authorized_keys | 4 ++ devops/tasks/config/nginx_toy_sba | 23 +++++++ devops/tasks/config/sync.csv | 7 +- devops/tasks/config/systemd_toy_sba | 11 ++++ devops/tasks/config/toy-sba.sh | 7 ++ devops/tasks/orchlib/aws.py | 3 +- devops/tasks/orchlib/config.py | 5 ++ devops/tasks/orchlib/ubuntu.py | 74 ++++++++++++++++++++-- devops/tasks/scripts/baseline_packages.fab | 20 +++++- devops/tasks/scripts/git_repos.fab | 38 +++++++++++ devops/tasks/scripts/python_venv.fab | 15 ++++- devops/tasks/tasks.py | 33 ++++++++++ 13 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 devops/tasks/config/authorized_keys create mode 100644 devops/tasks/config/nginx_toy_sba create mode 100644 devops/tasks/config/systemd_toy_sba create mode 100644 devops/tasks/config/toy-sba.sh create mode 100644 devops/tasks/scripts/git_repos.fab diff --git a/devops/requirements.txt b/devops/requirements.txt index fbb305608..c8fcea507 100644 --- a/devops/requirements.txt +++ b/devops/requirements.txt @@ -2,4 +2,4 @@ chevron boto3 pyyaml fabric - +unzip diff --git a/devops/tasks/config/authorized_keys b/devops/tasks/config/authorized_keys new file mode 100644 index 000000000..b25f6c109 --- /dev/null +++ b/devops/tasks/config/authorized_keys @@ -0,0 +1,4 @@ +Copy the following into: ~/.ssh/authorized_keys + +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/tasks/config/nginx_toy_sba b/devops/tasks/config/nginx_toy_sba new file mode 100644 index 000000000..5f9640c74 --- /dev/null +++ b/devops/tasks/config/nginx_toy_sba @@ -0,0 +1,23 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{hostname}}.{{domain}}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:3000/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + } +} \ No newline at end of file diff --git a/devops/tasks/config/sync.csv b/devops/tasks/config/sync.csv index e3592df21..4c294b12a 100644 --- a/devops/tasks/config/sync.csv +++ b/devops/tasks/config/sync.csv @@ -1,6 +1,7 @@ creds.yaml,root:root,644,/home/ubuntu/writing_observer/learning_observer/creds.yaml,"Learning Observer settings file" -nginx,root:root,644,/etc/nginx/sites-enabled/{hostname},"nginx site configuration" +nginx_toy_sba,root:root,644,/etc/nginx/sites-available/{hostname},"nginx site configuration" +authorized_keys,root:root,644,/home/ubuntu/authorized_keys,"nginx site configuration" passwd.lo,root:root,644,/home/ubuntu/writing_observer/learning_observer/passwd.lo,"(Generally blank) passwords file" -lo.sh,ubuntu:ubuntu,744,/home/ubuntu/writing_observer/learning_observer/lo.sh,"Script to start Learning Observer with a nice process name" -systemd,root:root,644,/etc/systemd/system/learning_observer.service,"Systemd init script" +toy-sba.sh,ubuntu:ubuntu,744,/home/ubuntu/toy-sba/toy-sba.sh,"Script to start toy-sba" +systemd_toy_sba,root:root,644,/etc/systemd/system/toy_sba.service,"Systemd init script for toy-sba" rsyslog.conf,root:root,644,/etc/rsyslog.d/learning_observer.conf,"rsyslog script (for stdout/stderr)" \ No newline at end of file diff --git a/devops/tasks/config/systemd_toy_sba b/devops/tasks/config/systemd_toy_sba new file mode 100644 index 000000000..b1e842676 --- /dev/null +++ b/devops/tasks/config/systemd_toy_sba @@ -0,0 +1,11 @@ +[Unit] +Description=toy-sba + +[Service] +ExecStart=/home/ubuntu/toy-sba/toy-sba.sh +Type=simple +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=toy_sba +User=ubuntu +Group=ubuntu \ No newline at end of file diff --git a/devops/tasks/config/toy-sba.sh b/devops/tasks/config/toy-sba.sh new file mode 100644 index 000000000..a010557b8 --- /dev/null +++ b/devops/tasks/config/toy-sba.sh @@ -0,0 +1,7 @@ +#!/usr/bin/bash + +# This is a script to start up toy-sba with it's own process +# name. This is convenient for being able to start / stop the process. + +cd /home/ubuntu/toy-sba +bash -c "npm run dev" >> /home/ubuntu/toy_sba.log 2>> /home/ubuntu/toy_sba.err diff --git a/devops/tasks/orchlib/aws.py b/devops/tasks/orchlib/aws.py index ce76b14ba..67c1b0f5c 100644 --- a/devops/tasks/orchlib/aws.py +++ b/devops/tasks/orchlib/aws.py @@ -20,6 +20,7 @@ r53 = boto3.client('route53') UBUNTU_20_04 = "ami-09e67e426f25ce0d7" +UBUNTU_24_04 = "ami-0e86e20dae9224db8" def create_instance(name): ''' @@ -69,7 +70,7 @@ def create_instance(name): # It doesn't correspond 1:1, but it's a good starting # point. response = ec2.create_instances( - ImageId=UBUNTU_20_04, + ImageId=UBUNTU_24_04, InstanceType='t2.small', BlockDeviceMappings=blockDeviceMappings, KeyName=orchlib.config.creds['aws_keyname'], diff --git a/devops/tasks/orchlib/config.py b/devops/tasks/orchlib/config.py index 330de195c..eb8f80874 100644 --- a/devops/tasks/orchlib/config.py +++ b/devops/tasks/orchlib/config.py @@ -17,6 +17,11 @@ "aws_security_group": "AWS security group (e.g. sg-012345abc)", "owner": "Your name", "email": "Your email", + "git_username": "Your git username", + "git_pac": "Your git personal access token", + "openai_url": "currently https://canadaeast.api.cognitive.microsoft.com", + "openai_deployment_id": "currently PBrostGPT4", + "openai_api_key": "your api key for the azure instance", "domain": "Domain name (e.g. learning-observer.org)", "flock-config": "Path to git repo where we'll store machine config.", "deploy-group": "Tag to identify all machines (typically, learning-observer)", diff --git a/devops/tasks/orchlib/ubuntu.py b/devops/tasks/orchlib/ubuntu.py index af4327d40..415270d3d 100644 --- a/devops/tasks/orchlib/ubuntu.py +++ b/devops/tasks/orchlib/ubuntu.py @@ -7,27 +7,89 @@ import orchlib.fabric_flock import orchlib.config - +import tempfile +import os def run_script(scriptfile): ''' Helper which executes a series of commands on set of machines ''' - script = open("scripts/{fn}.fab".format(fn=scriptfile)).read() + print("------------------------------------------") + + # Read the script file + script_path = "scripts/{fn}.fab".format(fn=scriptfile) + print("Running script: ", script_path) + try: + with open(script_path, 'r') as file: + script = file.read() + except FileNotFoundError: + print(f"Script file not found: {script_path}") + return + except Exception as e: + print(f"Error reading script file: {e}") + return + def run(*machines): group = orchlib.fabric_flock.machine_group(*machines) - + i = 0 for line in ['hostname'] + script.split("\n"): line = line.strip() + i = i + 1 if len(line) > 0 and line[0] != "#": - print(line) group.run(line) + print("------------------------------------------") return run update = run_script("update") baseline_packages = run_script("baseline_packages") python_venv = run_script("python_venv") +def install_git_repos(ip): + + git_username = orchlib.config.creds.get('git_username') + git_pac = orchlib.config.creds.get('git_pac') + openai_url = orchlib.config.creds.get('openai_url') + openai_deployment_id = orchlib.config.creds.get('openai_deployment_id') + openai_api_key = orchlib.config.creds.get('openai_api_key') + + if not git_username or not git_pac or not openai_url or not openai_deployment_id or not openai_api_key: + raise ValueError("Git/OpenAI credentials are not available in the config") + + # Path to the baseline_packages.fab file + fab_file_path = os.path.join(os.path.dirname(__file__), '..', 'scripts', 'git_repos.fab') + + # Read the content of the baseline_packages.fab file + with open(fab_file_path, 'r') as file: + content = file.read() + + # Replace placeholders with actual credentials + content = content.replace('{git_username}', git_username) + content = content.replace('{git_pac}', git_pac) + content = content.replace('{openai_url}', openai_url) + content = content.replace('{openai_deployment_id}', openai_deployment_id) + content = content.replace('{openai_api_key}', openai_api_key) + + # Write the updated content to a temporary file + temp_dir = os.path.join(os.path.dirname(__file__), '..', 'scripts') + with tempfile.NamedTemporaryFile(delete=False, suffix='.fab', dir=temp_dir) as temp_file: + temp_file.write(content.encode()) + temp_file_path = temp_file.name + + print(f"Running: {temp_file_path}") + temp_file_name = os.path.basename(temp_file_path) + temp_file_name = temp_file_name.replace('.fab', '') + + # Run the resulting script using run_script + print("Running baseline_packages script: " + temp_file_name) + + try: + run_script(temp_file_name)(ip) + except Exception as e: + print(f"An error occurred while running the script: {e}") + return None + finally: + # Clean up the temporary file + os.remove(temp_file_path) def reboot(machine): ''' @@ -47,7 +109,9 @@ def provision(ip): connect_kwargs={"key_filename": orchlib.config.creds['key_filename']} ) update() - baseline_packages() + + run_baseline_packages() + ###baseline_packages() python_venv() reboot() diff --git a/devops/tasks/scripts/baseline_packages.fab b/devops/tasks/scripts/baseline_packages.fab index fbd88ece4..f76de8f80 100644 --- a/devops/tasks/scripts/baseline_packages.fab +++ b/devops/tasks/scripts/baseline_packages.fab @@ -1,4 +1,18 @@ + cd -sudo apt-get -y install git ansible awscli -git clone https://github.com/ETS-Next-Gen/writing_observer.git -cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml + +sudo apt update +sudo apt-get install -y virtualenvwrapper +sudo apt-get -y install nginx +sudo apt-get install -y certbot python3-certbot-nginx +sudo nginx -s reload + +sudo apt-get -y install git ansible python3-pip python3-venv pipx + +# Ensure pipx binaries are available in the PATH +python3 -m pipx ensurepath + +# Install awscli using pipx +pipx install awscli + + diff --git a/devops/tasks/scripts/git_repos.fab b/devops/tasks/scripts/git_repos.fab new file mode 100644 index 000000000..dbd617757 --- /dev/null +++ b/devops/tasks/scripts/git_repos.fab @@ -0,0 +1,38 @@ +#Install npm for the toy-sba project +sudo apt-get -y install npm + +cd + +# Set system variables +export OPENAI_URL="{openai_url}" +export OPENAI_DEPLOYMENT_ID="{openai_deployment_id}" +export OPENAI_API_KEY="{openai_api_key}" + +# Check if the writing_observer directory exists before cloning +[ ! -d "writing_observer" ] && git clone https://github.com/ETS-Next-Gen/writing_observer.git || echo "Directory writing_observer already exists. Skipping clone." + +# Check if the toy-sba directory exists before cloning +[ ! -d "toy-sba" ] && git clone https://{git_username}:{git_pac}@github.com/ETS-Next-Gen/toy-sba.git || echo "Directory toy-sba already exists. Skipping clone." + + +#switch branches for writing_observer and toy-sba +# Navigate to the writing_observer repository directory + +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch pmitros/lo_event + +cd /home/ubuntu/toy-sba && git fetch && git switch writing_survey + +# Navigate to the lo_event module and install npm packages +cd /home/ubuntu/writing_observer/modules/lo_event && npm install + +# Install npm packages for toy-sba +cd /home/ubuntu/toy-sba +rm -rf node_modules package-lock.json +npm install --legacy-peer-deps +npm install @fortawesome/fontawesome-svg-core --legacy-peer-deps + +## PIOTR: what is the next statement for? It fails with the error: +## ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks. +cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml + + diff --git a/devops/tasks/scripts/python_venv.fab b/devops/tasks/scripts/python_venv.fab index 629c773eb..08b5eee5d 100644 --- a/devops/tasks/scripts/python_venv.fab +++ b/devops/tasks/scripts/python_venv.fab @@ -1,6 +1,19 @@ +sudo apt-get install -y virtualenvwrapper + +# Source virtualenvwrapper.sh +export WORKON_HOME=$HOME/.virtualenvs +source /usr/share/virtualenvwrapper/virtualenvwrapper.sh + +source ~/.bashrc +mkvirtualenv --version + +# Create a virtual environment +mkvirtualenv learning_observer + cd echo . /usr/share/virtualenvwrapper/virtualenvwrapper.sh >> ~/.profile -source ~/.profile; mkvirtualenv learning_observer +source ~/.profile; + echo workon learning_observer >> ~/.profile source ~/.profile; pip install --upgrade pip source ~/.profile; cd writing_observer/ ; pip install -r requirements.txt diff --git a/devops/tasks/tasks.py b/devops/tasks/tasks.py index 9339e993d..3c0d08037 100644 --- a/devops/tasks/tasks.py +++ b/devops/tasks/tasks.py @@ -52,6 +52,39 @@ def provision(c, machine_name): orchlib.ubuntu.update(ip) print("Baseline...") orchlib.ubuntu.baseline_packages(ip) + print("Git Repos...") + orchlib.ubuntu.install_git_repos(ip) + print("Venv...") + orchlib.ubuntu.python_venv(ip) + +@task +def initialize(c, machine_name): + ''' + Set up a baseline image with all the packages needed for + Learning Observer. Note that this will **not** configure + the machine. + ''' + print("Provisioning...") + machine_info = orchlib.aws.create_instance(machine_name) + print("Updating...") + ip = machine_info.public_ip_address + print("DNS....") + orchlib.aws.register_dns(machine_name, orchlib.config.creds['domain'], ip) + print("IP", ip) + orchlib.ubuntu.update(ip) + +@task +def baseline(c, ip): + print("Baseline...") + orchlib.ubuntu.baseline_packages(ip) + +@task +def gitrepos(c, ip): + print("Git Repos...") + orchlib.ubuntu.install_git_repos(ip) + +@task +def venv(c, ip): print("Venv...") orchlib.ubuntu.python_venv(ip) From 71de24dc7a0a8655e11cb09c633abf84140b64ca Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Thu, 12 Sep 2024 14:14:31 -0400 Subject: [PATCH 02/16] update to readme --- devops/tasks/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/devops/tasks/README.md b/devops/tasks/README.md index 156977314..03766d4bc 100644 --- a/devops/tasks/README.md +++ b/devops/tasks/README.md @@ -89,6 +89,13 @@ inv certbot [machine] inv downloadconfig [machine] ``` +inv provision [machine] can be run step by step as well, using the commands + +sudo inv initialize [machine] +inv baseline [ip address] +inv gitrepos [ip address] +inv venv [ip address] + From there, edit configuration files in `config` and to update the machine to a new version, run From 138a40742c20ce72f5db9cdc0bbd8e267cb69e2d Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 16 Sep 2024 11:51:32 -0400 Subject: [PATCH 03/16] ansible change, pushing to toy-sba server for testing --- devops/ansible/local.yaml | 3 +-- devops/ansible/tasks/writing-apt.yaml | 5 ++++- devops/tasks/config/toy-sba.sh | 2 +- devops/tasks/orchlib/config.py | 4 ++-- devops/tasks/scripts/baseline_packages.fab | 24 +++++++++++++++------- devops/tasks/scripts/git_repos.fab | 8 ++------ 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/devops/ansible/local.yaml b/devops/ansible/local.yaml index 3d502ca93..1ed450fdc 100644 --- a/devops/ansible/local.yaml +++ b/devops/ansible/local.yaml @@ -2,5 +2,4 @@ hosts: localhost connection: local tasks: - - include: tasks/writing-apt.yaml - + - include_tasks: tasks/writing-apt.yaml diff --git a/devops/ansible/tasks/writing-apt.yaml b/devops/ansible/tasks/writing-apt.yaml index e00d6b134..4beb967c8 100644 --- a/devops/ansible/tasks/writing-apt.yaml +++ b/devops/ansible/tasks/writing-apt.yaml @@ -54,4 +54,7 @@ - certbot - apache2-utils - fcgiwrap - - python3-certbot-nginx \ No newline at end of file + - python3-certbot-nginx + - npm + - virtualenvwrapper + - awscli \ No newline at end of file diff --git a/devops/tasks/config/toy-sba.sh b/devops/tasks/config/toy-sba.sh index a010557b8..d3c018ba0 100644 --- a/devops/tasks/config/toy-sba.sh +++ b/devops/tasks/config/toy-sba.sh @@ -4,4 +4,4 @@ # name. This is convenient for being able to start / stop the process. cd /home/ubuntu/toy-sba -bash -c "npm run dev" >> /home/ubuntu/toy_sba.log 2>> /home/ubuntu/toy_sba.err +bash -c "sudo -E npm run dev" >> /home/ubuntu/toy_sba.log 2>> /home/ubuntu/toy_sba.err diff --git a/devops/tasks/orchlib/config.py b/devops/tasks/orchlib/config.py index eb8f80874..93eeb38c2 100644 --- a/devops/tasks/orchlib/config.py +++ b/devops/tasks/orchlib/config.py @@ -19,8 +19,8 @@ "email": "Your email", "git_username": "Your git username", "git_pac": "Your git personal access token", - "openai_url": "currently https://canadaeast.api.cognitive.microsoft.com", - "openai_deployment_id": "currently PBrostGPT4", + "openai_url": " URL for OPENAI API", + "openai_deployment_id": "OPEN AI deployment ID", "openai_api_key": "your api key for the azure instance", "domain": "Domain name (e.g. learning-observer.org)", "flock-config": "Path to git repo where we'll store machine config.", diff --git a/devops/tasks/scripts/baseline_packages.fab b/devops/tasks/scripts/baseline_packages.fab index f76de8f80..f90395b83 100644 --- a/devops/tasks/scripts/baseline_packages.fab +++ b/devops/tasks/scripts/baseline_packages.fab @@ -2,17 +2,27 @@ cd sudo apt update -sudo apt-get install -y virtualenvwrapper -sudo apt-get -y install nginx -sudo apt-get install -y certbot python3-certbot-nginx -sudo nginx -s reload +##sudo apt-get install -y virtualenvwrapper +##sudo apt-get -y install nginx +##sudo apt-get install -y certbot python3-certbot-nginx +##sudo nginx -s reload +##sudo apt-get -y install git ansible python3-pip python3-venv pipx -sudo apt-get -y install git ansible python3-pip python3-venv pipx +#sudo apt-get -y install git ansible awscli +sudo apt-get -y install git ansible + +# Check if the writing_observer directory exists before cloning + +[ ! -d "writing_observer" ] && git clone https://github.com/ETS-Next-Gen/writing_observer.git || echo "Directory writing_observer already exists. Skipping clone." + +## PIOTR: what is the next statement for? It fails with the error: +## ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks. +cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml # Ensure pipx binaries are available in the PATH -python3 -m pipx ensurepath +#python3 -m pipx ensurepath # Install awscli using pipx -pipx install awscli +#pipx install awscli diff --git a/devops/tasks/scripts/git_repos.fab b/devops/tasks/scripts/git_repos.fab index dbd617757..42785b89c 100644 --- a/devops/tasks/scripts/git_repos.fab +++ b/devops/tasks/scripts/git_repos.fab @@ -8,9 +8,6 @@ export OPENAI_URL="{openai_url}" export OPENAI_DEPLOYMENT_ID="{openai_deployment_id}" export OPENAI_API_KEY="{openai_api_key}" -# Check if the writing_observer directory exists before cloning -[ ! -d "writing_observer" ] && git clone https://github.com/ETS-Next-Gen/writing_observer.git || echo "Directory writing_observer already exists. Skipping clone." - # Check if the toy-sba directory exists before cloning [ ! -d "toy-sba" ] && git clone https://{git_username}:{git_pac}@github.com/ETS-Next-Gen/toy-sba.git || echo "Directory toy-sba already exists. Skipping clone." @@ -29,10 +26,9 @@ cd /home/ubuntu/writing_observer/modules/lo_event && npm install cd /home/ubuntu/toy-sba rm -rf node_modules package-lock.json npm install --legacy-peer-deps +npm install react-diff-components --legacy-peer-deps npm install @fortawesome/fontawesome-svg-core --legacy-peer-deps -## PIOTR: what is the next statement for? It fails with the error: -## ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks. -cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml + From 3ee211861e5687969531a71cf60ae410dbe91e03 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 23 Sep 2024 20:41:47 -0400 Subject: [PATCH 04/16] moved code to ansible yaml files --- .../ansible/files_to_upload/authorized_keys | 4 + devops/ansible/files_to_upload/htpasswd | 1 + devops/ansible/files_to_upload/nginx | 40 +++++++ devops/ansible/files_to_upload/nginx_toy_sba | 28 +++++ devops/ansible/tasks/baseline.yaml | 80 +++++++++++++ devops/ansible/tasks/copy_files.yaml | 51 +++++++++ devops/ansible/tasks/install_repos.yaml | 40 +++++++ .../tasks/os_environment_variables.yaml | 17 +++ devops/ansible/tasks/reboot.yaml | 10 ++ devops/ansible/tasks/test.yaml | 34 ++++++ devops/ansible/tasks/toy-sba.yaml | 108 ++++++++++++++++++ devops/ansible/tasks/writing-apt.yaml | 42 ++++--- devops/tasks/README.md | 23 +++- devops/tasks/config/nginx_toy_sba | 4 + devops/tasks/hosts.ini | 3 + devops/tasks/orchlib/aws.py | 12 +- devops/tasks/orchlib/config.py | 2 +- devops/tasks/scripts/baseline_packages.fab | 6 +- devops/tasks/settings/LO_FILES.YAML | 6 + devops/tasks/settings/TOY_REPOS.YAML | 9 ++ devops/tasks/settings/TOY_SBA_FILES.YAML | 6 + devops/tasks/settings/WO_REPOS.YAML | 5 + devops/tasks/tasks.py | 7 ++ package.json | 3 + 24 files changed, 520 insertions(+), 21 deletions(-) create mode 100644 devops/ansible/files_to_upload/authorized_keys create mode 100644 devops/ansible/files_to_upload/htpasswd create mode 100644 devops/ansible/files_to_upload/nginx create mode 100644 devops/ansible/files_to_upload/nginx_toy_sba create mode 100644 devops/ansible/tasks/baseline.yaml create mode 100644 devops/ansible/tasks/copy_files.yaml create mode 100644 devops/ansible/tasks/install_repos.yaml create mode 100644 devops/ansible/tasks/os_environment_variables.yaml create mode 100644 devops/ansible/tasks/reboot.yaml create mode 100644 devops/ansible/tasks/test.yaml create mode 100644 devops/ansible/tasks/toy-sba.yaml create mode 100644 devops/tasks/hosts.ini create mode 100644 devops/tasks/settings/LO_FILES.YAML create mode 100644 devops/tasks/settings/TOY_REPOS.YAML create mode 100644 devops/tasks/settings/TOY_SBA_FILES.YAML create mode 100644 devops/tasks/settings/WO_REPOS.YAML diff --git a/devops/ansible/files_to_upload/authorized_keys b/devops/ansible/files_to_upload/authorized_keys new file mode 100644 index 000000000..b25f6c109 --- /dev/null +++ b/devops/ansible/files_to_upload/authorized_keys @@ -0,0 +1,4 @@ +Copy the following into: ~/.ssh/authorized_keys + +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/ansible/files_to_upload/htpasswd b/devops/ansible/files_to_upload/htpasswd new file mode 100644 index 000000000..00db81b64 --- /dev/null +++ b/devops/ansible/files_to_upload/htpasswd @@ -0,0 +1 @@ +coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30 diff --git a/devops/ansible/files_to_upload/nginx b/devops/ansible/files_to_upload/nginx new file mode 100644 index 000000000..3067c6347 --- /dev/null +++ b/devops/ansible/files_to_upload/nginx @@ -0,0 +1,40 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{ hostname }}.{{ server_domain }}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:8888/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + } + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + if ($request_method = OPTIONS ) { + return 200; + } + } +} \ No newline at end of file diff --git a/devops/ansible/files_to_upload/nginx_toy_sba b/devops/ansible/files_to_upload/nginx_toy_sba new file mode 100644 index 000000000..1961236d8 --- /dev/null +++ b/devops/ansible/files_to_upload/nginx_toy_sba @@ -0,0 +1,28 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{hostname}}.{{server_domain}}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:3000/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + auth_basic "Restricted Area"; + auth_basic_user_file /etc/nginx/htpasswd; + + try_files $uri $uri/ =404; + } +} \ No newline at end of file diff --git a/devops/ansible/tasks/baseline.yaml b/devops/ansible/tasks/baseline.yaml new file mode 100644 index 000000000..436e2c790 --- /dev/null +++ b/devops/ansible/tasks/baseline.yaml @@ -0,0 +1,80 @@ +- name: Provision writing analysis server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - apt: upgrade=dist update_cache=yes + + - name: Basic utils + apt: name={{ item }} + with_items: + - curl + - emacs + - git + - git-core + - links + - lynx + - mosh + - nmap + - whois + - screen + - wipe + - build-essential + - net-tools + + # We don't need all of this per se, but it's convenient. If nothing + # else, it gives prereqs for `pip` + - name: Install Python Packages + apt: + name: "{{ item }} " + state: present + with_items: + - ipython3 + - libxml2-dev + - libxslt1-dev + - python3-boto + - python3-bson + - python3-dev + - python3-matplotlib + - python3-numpy + - python3-pandas + - python3-pip + - python3-scipy + - python3-setuptools + - python3-sklearn + - virtualenvwrapper + - libjpeg-dev + - python3-opencv + - python3-virtualenv + - python3-aiohttp + - python3-aiohttp-cors + - python3-tornado + - python3-venv + - python3-yaml + - python3-asyncpg + - python3-bcrypt + + - name: Install server packages + apt: + name: "{{ item }}" + state: present + loop: + - redis + - nginx + - certbot + - apache2-utils + - fcgiwrap + - python3-certbot-nginx + - virtualenvwrapper + + - name: Install pip + apt: + name: python3-pip + state: present + + - name: Create a virtual environment for AWS CLI + command: python3 -m venv /opt/learning-observer + + - name: Install AWS CLI using pip in virtual environment + command: /opt/learning-observer/bin/pip install awscli diff --git a/devops/ansible/tasks/copy_files.yaml b/devops/ansible/tasks/copy_files.yaml new file mode 100644 index 000000000..d5fdf1acf --- /dev/null +++ b/devops/ansible/tasks/copy_files.yaml @@ -0,0 +1,51 @@ +- name: Copy files to server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + vars_files: + - ../../tasks/settings/LO_FILES.yaml + - ../../tasks/settings/CREDS.yaml + + tasks: + - name: Ensure destination directories exist + file: + path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" + state: directory + loop: "{{ files }}" + + - name: Copy files to server + template: + src: "{{ item.src }}" + dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + owner: root + group: root + mode: '0644' + loop: "{{ files }}" + vars: + hostname: "{{ inventory_hostname }}" + server_domain: "{{ domain }}" + nginx_root_options: "" + + - name: Delete the symlink /etc/nginx/sites-enabled/default + file: + path: /etc/nginx/sites-enabled/default + state: absent + + - name: Delete the symlink /etc/nginx/sites-available/default + file: + path: /etc/nginx/sites-enabled/default + state: absent + + - name: Create a symlink for the nginx file + file: + src: /etc/nginx/sites-available/{{ inventory_hostname }} + dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} + state: link + + - name: Restart nginx + service: + name: nginx + state: restarted + become: yes + \ No newline at end of file diff --git a/devops/ansible/tasks/install_repos.yaml b/devops/ansible/tasks/install_repos.yaml new file mode 100644 index 000000000..f4e243360 --- /dev/null +++ b/devops/ansible/tasks/install_repos.yaml @@ -0,0 +1,40 @@ +- name: Install Git Repositories + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Install Git + apt: + name: git + state: present + + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Read repository configuration + include_vars: + file: ../../tasks/settings/WO_REPOS.YAML + +# - name: Debug git_username variable +# debug: +# var: git_username + +# - name: Debug git_pac variable +# debug: +# var: git_pac + +# - name: Debug repos variable +# debug: +# var: repos + + - name: Clone Git repositories + git: + repo: "https://{{ git_username }}:{{ git_pac }}@{{ item.repo | regex_replace('https://', '') }}" + dest: "{{ item.dest }}" + clone: yes + update: yes + version: "{{ item.branch | default('master') }}" + force: yes + loop: "{{ repos }}" \ No newline at end of file diff --git a/devops/ansible/tasks/os_environment_variables.yaml b/devops/ansible/tasks/os_environment_variables.yaml new file mode 100644 index 000000000..04e13517a --- /dev/null +++ b/devops/ansible/tasks/os_environment_variables.yaml @@ -0,0 +1,17 @@ +- name: Set Environment Variables + hosts: all + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read environment configuration + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Populate /etc/environment + lineinfile: + path: "/etc/environment" + state: present + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + with_items: "{{ os_environment }}" \ No newline at end of file diff --git a/devops/ansible/tasks/reboot.yaml b/devops/ansible/tasks/reboot.yaml new file mode 100644 index 000000000..409439fa4 --- /dev/null +++ b/devops/ansible/tasks/reboot.yaml @@ -0,0 +1,10 @@ +- name: Reboot the server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Reboot the server + reboot: + msg: "Reboot initiated by Ansible" + reboot_timeout: 600 \ No newline at end of file diff --git a/devops/ansible/tasks/test.yaml b/devops/ansible/tasks/test.yaml new file mode 100644 index 000000000..718ef3043 --- /dev/null +++ b/devops/ansible/tasks/test.yaml @@ -0,0 +1,34 @@ +- name: Copy files to server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + vars_files: + - ../../tasks/settings/CREDS.yaml + - ../../tasks/settings/TOY_SBA_FILES.yaml + + tasks: + - name: Ensure destination directories exist + file: + path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" + state: directory + loop: "{{ files }}" + + - name: Copy files to server + template: + src: "{{ item.src }}" + dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + owner: root + group: root + mode: '0644' + loop: "{{ files }}" + vars: + hostname: "{{ inventory_hostname }}" + server_domain: "{{ domain }}" + nginx_root_options: "" + + - name: Restart nginx + service: + name: nginx + state: restarted + become: yes diff --git a/devops/ansible/tasks/toy-sba.yaml b/devops/ansible/tasks/toy-sba.yaml new file mode 100644 index 000000000..c925c3d34 --- /dev/null +++ b/devops/ansible/tasks/toy-sba.yaml @@ -0,0 +1,108 @@ +- name: Additional provisions for toy-sba server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + vars_files: + - ../../tasks/settings/TOY_SBA_FILES.yaml + - ../../tasks/settings/CREDS.yaml + - ../../tasks/settings/TOY_REPOS.YAML + + tasks: + - name: Populate /etc/environment + lineinfile: + path: "/etc/environment" + state: present + regexp: "^{{ item.key }}=" + line: "{{ item.key }}={{ item.value }}" + with_items: "{{ os_environment }}" + + - name: Clone Git repositories + git: + repo: "https://{{ git_username }}:{{ git_pac }}@{{ item.repo | regex_replace('https://', '') }}" + dest: "{{ item.dest }}" + clone: yes + update: yes + version: "{{ item.branch | default('master') }}" + force: yes + loop: "{{ repos }}" + + - name: Install npm + apt: + name: npm + state: present + + - name: Install pm2 using npm + npm: + name: pm2 + global: yes + + - name: Run npm install for lo_event + shell: npm install + args: + chdir: /home/ubuntu/writing_observer/modules/lo_event + + - name: Run npm install for toy-sba + shell: npm install --legacy-peer-deps + args: + chdir: /home/ubuntu/toy-sba + + - name: Ensure destination directories exist + file: + path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" + state: directory + loop: "{{ files }}" + + - name: Copy files to server + template: + src: "{{ item.src }}" + dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + owner: root + group: root + mode: '0644' + loop: "{{ files }}" + vars: + hostname: "{{ inventory_hostname }}" + server_domain: "{{ domain }}" + nginx_root_options: "" + + - name: Create a symlink for the nginx file + file: + src: /etc/nginx/sites-available/{ inventory_hostname }} + dest: /etc/nginx/sites-enabled/{ inventory_hostname }} + state: link + + - name: Restart nginx + service: + name: nginx + state: restarted + become: yes + + - name: Check if toy-sba is running with pm2 + shell: pm2 describe toy-sba + register: pm2_toy_sba_running + ignore_errors: true + failed_when: false + + - name: Debug pm2_toy_sba_running + debug: + msg: "pm2_toy_sba_running.rc is {{ pm2_toy_sba_running.rc }}" + + - name: Start toy-sba with pm2 if not running + shell: pm2 start npm --name "toy-sba" -- run dev + when: pm2_toy_sba_running.rc != 0 + args: + chdir: /home/ubuntu/toy-sba + become: yes + + - name: Wait for 5 seconds before restarting toy-sba + wait_for: + timeout: 5 + when: pm2_toy_sba_running.rc == 0 + + - name: Restart toy-sba with pm2 if running + shell: pm2 restart toy-sba + when: pm2_toy_sba_running.rc == 0 + args: + chdir: /home/ubuntu/toy-sba + become: yes diff --git a/devops/ansible/tasks/writing-apt.yaml b/devops/ansible/tasks/writing-apt.yaml index 4beb967c8..5032f1234 100644 --- a/devops/ansible/tasks/writing-apt.yaml +++ b/devops/ansible/tasks/writing-apt.yaml @@ -19,8 +19,10 @@ # We don't need all of this per se, but it's convenient. If nothing # else, it gives prereqs for `pip` -- name: Python - apt: name={{ item }} +- name: Install Python Packages + apt: + name: "{{ item }} " + state: present with_items: - ipython3 - libxml2-dev @@ -42,19 +44,31 @@ - python3-aiohttp - python3-aiohttp-cors - python3-tornado + - python3-venv - python3-yaml - python3-asyncpg - python3-bcrypt -- name: Server - apt: name={{ item }} - with_items: - - redis - - nginx - - certbot - - apache2-utils - - fcgiwrap - - python3-certbot-nginx - - npm - - virtualenvwrapper - - awscli \ No newline at end of file +- name: Install server packages + apt: + name: "{{ item }}" + state: present + loop: + - redis + - nginx + - certbot + - apache2-utils + - fcgiwrap + - python3-certbot-nginx + - virtualenvwrapper + +- name: Install pip + apt: + name: python3-pip + state: present + +- name: Create a virtual environment for AWS CLI + command: python3 -m venv /opt/learning-observer + +- name: Install AWS CLI using pip in virtual environment + command: /opt/learning-observer/bin/pip install awscli \ No newline at end of file diff --git a/devops/tasks/README.md b/devops/tasks/README.md index 03766d4bc..37a0e83e8 100644 --- a/devops/tasks/README.md +++ b/devops/tasks/README.md @@ -128,4 +128,25 @@ design: atomic commits at the end (e.g. download to a temporary dir, and move right before the commit. * However, it is possible to reverse-engineered exactly what happened, - roughly when. This is good enough for now. \ No newline at end of file + roughly when. This is good enough for now. + + + @@@@PB + setup hosts.ini file, and run the following command to configure the remote server: + (from inside devops/tasks folder) + + sudo inv initialize [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/baseline.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/install_repos.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/copy_files.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit [machine] + + ###If setting up + ansible-playbook -i hosts.ini ../ansible/tasks/toy-sba.yaml --limit [machine] + + inv certbot [machine] + inv downloadconfig [machine] + + + inv certbot coglabs + inv downloadconfig coglabs \ No newline at end of file diff --git a/devops/tasks/config/nginx_toy_sba b/devops/tasks/config/nginx_toy_sba index 5f9640c74..389af3210 100644 --- a/devops/tasks/config/nginx_toy_sba +++ b/devops/tasks/config/nginx_toy_sba @@ -19,5 +19,9 @@ server { add_header "Access-Control-Allow-Origin" *; add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + # Enable HTTP Basic Authentication + auth_basic "Restricted Content"; + auth_basic_user_file /etc/nginx/.htpasswd; } } \ No newline at end of file diff --git a/devops/tasks/hosts.ini b/devops/tasks/hosts.ini new file mode 100644 index 000000000..ce16bcb78 --- /dev/null +++ b/devops/tasks/hosts.ini @@ -0,0 +1,3 @@ +[servers] +brostpS2 ansible_host=3.84.57.3 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file diff --git a/devops/tasks/orchlib/aws.py b/devops/tasks/orchlib/aws.py index 67c1b0f5c..a6adad5f8 100644 --- a/devops/tasks/orchlib/aws.py +++ b/devops/tasks/orchlib/aws.py @@ -28,11 +28,11 @@ def create_instance(name): ''' blockDeviceMappings = [ { - "DeviceName": "/dev/xvda", + "DeviceName": "/dev/sda1", ###xvda", "Ebs": { "DeleteOnTermination": True, - "VolumeSize": 32, - "VolumeType": "gp2" + "VolumeSize": 16, + "VolumeType": "gp3" } } ] @@ -50,6 +50,10 @@ def create_instance(name): { 'Key': 'deploy-group', 'Value': orchlib.config.creds['deploy-group'] + }, + { + 'Key': 'Patch Group', + 'Value': 'pet' } ] @@ -71,7 +75,7 @@ def create_instance(name): # point. response = ec2.create_instances( ImageId=UBUNTU_24_04, - InstanceType='t2.small', + InstanceType='t3.small', BlockDeviceMappings=blockDeviceMappings, KeyName=orchlib.config.creds['aws_keyname'], MinCount=1, diff --git a/devops/tasks/orchlib/config.py b/devops/tasks/orchlib/config.py index 93eeb38c2..84233b1db 100644 --- a/devops/tasks/orchlib/config.py +++ b/devops/tasks/orchlib/config.py @@ -18,7 +18,7 @@ "owner": "Your name", "email": "Your email", "git_username": "Your git username", - "git_pac": "Your git personal access token", + "git_pac": "Your git personal access token; See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens", "openai_url": " URL for OPENAI API", "openai_deployment_id": "OPEN AI deployment ID", "openai_api_key": "your api key for the azure instance", diff --git a/devops/tasks/scripts/baseline_packages.fab b/devops/tasks/scripts/baseline_packages.fab index f90395b83..c6e2d7837 100644 --- a/devops/tasks/scripts/baseline_packages.fab +++ b/devops/tasks/scripts/baseline_packages.fab @@ -12,9 +12,11 @@ sudo apt update sudo apt-get -y install git ansible # Check if the writing_observer directory exists before cloning - [ ! -d "writing_observer" ] && git clone https://github.com/ETS-Next-Gen/writing_observer.git || echo "Directory writing_observer already exists. Skipping clone." +# Navigate to the writing_observer repository directory +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch devops_toy_sba + ## PIOTR: what is the next statement for? It fails with the error: ## ERROR! [DEPRECATED]: ansible.builtin.include has been removed. Use include_tasks or import_tasks instead. This feature was removed from ansible-core in a release after 2023-05-16. Please update your playbooks. cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml @@ -25,4 +27,6 @@ cd writing_observer/devops/ansible ; sudo ansible-playbook local.yaml # Install awscli using pipx #pipx install awscli +# Navigate to the writing_observer repository directory +cd /home/ubuntu/writing_observer && pwd && git fetch && git switch pmitros/lo_event diff --git a/devops/tasks/settings/LO_FILES.YAML b/devops/tasks/settings/LO_FILES.YAML new file mode 100644 index 000000000..0d6697ec1 --- /dev/null +++ b/devops/tasks/settings/LO_FILES.YAML @@ -0,0 +1,6 @@ +files: + - src: ../../ansible/files_to_upload/nginx + dest: /etc/nginx/sites-available/{hostname} + - src: ../../ansible/files_to_upload/authorized_keys + dest: /home/ubuntu/authorized_keys + \ No newline at end of file diff --git a/devops/tasks/settings/TOY_REPOS.YAML b/devops/tasks/settings/TOY_REPOS.YAML new file mode 100644 index 000000000..9d7784e76 --- /dev/null +++ b/devops/tasks/settings/TOY_REPOS.YAML @@ -0,0 +1,9 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: pmitros/lo_event + - name: Toy-SBA + repo: https://github.com/ETS-Next-Gen/toy-sba.git + dest: /home/ubuntu/toy-sba + branch: writing_survey \ No newline at end of file diff --git a/devops/tasks/settings/TOY_SBA_FILES.YAML b/devops/tasks/settings/TOY_SBA_FILES.YAML new file mode 100644 index 000000000..9c87b1548 --- /dev/null +++ b/devops/tasks/settings/TOY_SBA_FILES.YAML @@ -0,0 +1,6 @@ +files: + - src: ../../ansible/files_to_upload/nginx_toy_sba + dest: /etc/nginx/sites-available/{hostname} + - src: ../../ansible/files_to_upload/htpasswd + dest: /etc/nginx/htpasswd + \ No newline at end of file diff --git a/devops/tasks/settings/WO_REPOS.YAML b/devops/tasks/settings/WO_REPOS.YAML new file mode 100644 index 000000000..72792054b --- /dev/null +++ b/devops/tasks/settings/WO_REPOS.YAML @@ -0,0 +1,5 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: master \ No newline at end of file diff --git a/devops/tasks/tasks.py b/devops/tasks/tasks.py index 3c0d08037..33e34e333 100644 --- a/devops/tasks/tasks.py +++ b/devops/tasks/tasks.py @@ -72,6 +72,13 @@ def initialize(c, machine_name): orchlib.aws.register_dns(machine_name, orchlib.config.creds['domain'], ip) print("IP", ip) orchlib.ubuntu.update(ip) + + # Write to hosts.ini + hosts_ini_path = '../settings/hosts.ini' + line_to_write = f"{machine_name} ansible_host={ip} ansible_user=ubuntu ansible_ssh_private_key_file={orchlib.config.creds['key_filename']}\n" + with open(hosts_ini_path, 'a') as hosts_file: + hosts_file.write(line_to_write) + print(f"Added {machine_name} to {hosts_ini_path}") @task def baseline(c, ip): diff --git a/package.json b/package.json index b5d78de21..467d36f3c 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,8 @@ "stylelint": "^15.5.0", "stylelint-config-standard": "^33.0.0", "stylelint-scss": "^4.6.0" + }, + "dependencies": { + "nodemon": "^3.1.5" } } From 9b79ef8a83e750aa264e3b074b332194cbe13fad Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Thu, 26 Sep 2024 16:19:36 -0400 Subject: [PATCH 05/16] pushing ansible changes --- devops/ansible/README.md | 41 +++++++ devops/ansible/default_flock/files.yaml | 5 + .../default_flock/files/authorized_keys | 2 + devops/ansible/default_flock/files/htpasswd | 1 + .../default_flock/files}/lo.sh | 0 devops/ansible/default_flock/files/nginx | 40 +++++++ .../default_flock/files}/passwd.lo | 0 .../default_flock/files}/systemd | 0 .../default_flock/yaml/ADDITIONAL_TASKS.YAML | 22 ++++ .../yaml/BASELINE.YAML} | 9 +- devops/ansible/default_flock/yaml/FILES.YAML | 12 ++ .../default_flock/yaml/OS_ENV_VARS.YAML | 4 + devops/ansible/default_flock/yaml/REPOS.YAML | 5 + devops/ansible/default_flock/yamls.yaml | 7 ++ .../ansible/files_to_upload/authorized_keys | 2 - devops/ansible/hosts.ini | 3 + devops/ansible/tasks/A_setup_flock.yaml | 72 ++++++++++++ devops/ansible/tasks/B_install_baseline.yaml | 12 ++ ...nstall_repos.yaml => C_install_repos.yaml} | 26 ++--- .../tasks/{test.yaml => D_copy_files.yaml} | 26 +++-- ... => E_setup_os_environment_variables.yaml} | 11 +- .../ansible/tasks/F_run_additional_tasks.yaml | 17 +++ devops/ansible/tasks/G_download_config.yaml | 34 ++++++ devops/ansible/tasks/copy_files.yaml | 51 --------- devops/ansible/tasks/master_playbook.yaml | 14 +++ devops/ansible/tasks/toy-sba.yaml | 108 ------------------ devops/ansible/tasks/writing-apt.yaml | 74 ------------ devops/tasks/README.md | 6 +- devops/tasks/config/creds.yaml | 33 ------ .../authorized_keys | 0 .../hostname | 0 .../init.d | 0 devops/tasks/config_NO_LONGER_NEEDED/lo.sh | 9 ++ .../{config => config_NO_LONGER_NEEDED}/nginx | 0 .../nginx_toy_sba | 0 .../tasks/config_NO_LONGER_NEEDED/passwd.lo | 0 .../postuploads | 0 .../rsyslog.conf | 0 .../sync.csv | 0 devops/tasks/config_NO_LONGER_NEEDED/systemd | 11 ++ .../systemd_toy_sba | 0 .../toy-sba.sh | 0 devops/tasks/hosts.ini | 3 +- devops/tasks/orchlib/aws.py | 8 +- devops/tasks/orchlib/config.py | 16 +-- devops/tasks/orchlib/logger.py | 2 +- devops/tasks/tasks.py | 5 +- 47 files changed, 366 insertions(+), 325 deletions(-) create mode 100644 devops/ansible/README.md create mode 100644 devops/ansible/default_flock/files.yaml create mode 100644 devops/ansible/default_flock/files/authorized_keys create mode 100644 devops/ansible/default_flock/files/htpasswd rename devops/{tasks/config => ansible/default_flock/files}/lo.sh (100%) create mode 100644 devops/ansible/default_flock/files/nginx rename devops/{tasks/config => ansible/default_flock/files}/passwd.lo (100%) rename devops/{tasks/config => ansible/default_flock/files}/systemd (100%) create mode 100644 devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML rename devops/ansible/{tasks/baseline.yaml => default_flock/yaml/BASELINE.YAML} (90%) create mode 100644 devops/ansible/default_flock/yaml/FILES.YAML create mode 100644 devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML create mode 100644 devops/ansible/default_flock/yaml/REPOS.YAML create mode 100644 devops/ansible/default_flock/yamls.yaml create mode 100644 devops/ansible/hosts.ini create mode 100644 devops/ansible/tasks/A_setup_flock.yaml create mode 100644 devops/ansible/tasks/B_install_baseline.yaml rename devops/ansible/tasks/{install_repos.yaml => C_install_repos.yaml} (63%) rename devops/ansible/tasks/{test.yaml => D_copy_files.yaml} (57%) rename devops/ansible/tasks/{os_environment_variables.yaml => E_setup_os_environment_variables.yaml} (58%) create mode 100644 devops/ansible/tasks/F_run_additional_tasks.yaml create mode 100644 devops/ansible/tasks/G_download_config.yaml delete mode 100644 devops/ansible/tasks/copy_files.yaml create mode 100644 devops/ansible/tasks/master_playbook.yaml delete mode 100644 devops/ansible/tasks/toy-sba.yaml delete mode 100644 devops/ansible/tasks/writing-apt.yaml delete mode 100644 devops/tasks/config/creds.yaml rename devops/tasks/{config => config_NO_LONGER_NEEDED}/authorized_keys (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/hostname (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/init.d (100%) create mode 100644 devops/tasks/config_NO_LONGER_NEEDED/lo.sh rename devops/tasks/{config => config_NO_LONGER_NEEDED}/nginx (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/nginx_toy_sba (100%) create mode 100644 devops/tasks/config_NO_LONGER_NEEDED/passwd.lo rename devops/tasks/{config => config_NO_LONGER_NEEDED}/postuploads (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/rsyslog.conf (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/sync.csv (100%) create mode 100644 devops/tasks/config_NO_LONGER_NEEDED/systemd rename devops/tasks/{config => config_NO_LONGER_NEEDED}/systemd_toy_sba (100%) rename devops/tasks/{config => config_NO_LONGER_NEEDED}/toy-sba.sh (100%) diff --git a/devops/ansible/README.md b/devops/ansible/README.md new file mode 100644 index 000000000..1ea808a65 --- /dev/null +++ b/devops/ansible/README.md @@ -0,0 +1,41 @@ +This is a set of ansible playbooks to setup a new server, currently reliant on the python tasks to assist in the server creation. + +To setup a new server, do the following: + + cd ../tasks + sudo inv initialize [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=[machine]" + ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit [machine] + ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit [machine] + + ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine] + inv inv certbot [machine] + ansible-playbook -i hosts.ini ./tasks/reboot.yaml --limit [machine] + + ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit [machine] + + TO DO: replace the hosts.ini with a python script that returns inv list output in correct format + + Example for daclassroom: + + sudo inv initialize daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom" + + ansible-playbook -i hosts.ini ../ansible/tasks/B_install_baseline.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/C_install_repos.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/D_copy_files.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/E_setup_os_environment_variables.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/F_run_additional_tasks.yaml --limit daclassroom + inv certbot daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom + + alternately, we can just use the master playbook to run steps B-F in one go: + sudo inv initialize daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/A_setup_flock.yaml --extra-vars "target_server=daclassroom" + + ansible-playbook -i hosts.ini ../ansible/tasks/master_playbook.yaml --limit daclassroom + inv certbot daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom diff --git a/devops/ansible/default_flock/files.yaml b/devops/ansible/default_flock/files.yaml new file mode 100644 index 000000000..42627acb2 --- /dev/null +++ b/devops/ansible/default_flock/files.yaml @@ -0,0 +1,5 @@ +files: + - name: authorized_keys + - name: htpasswd + - name: nginx + \ No newline at end of file diff --git a/devops/ansible/default_flock/files/authorized_keys b/devops/ansible/default_flock/files/authorized_keys new file mode 100644 index 000000000..46d2e8ba5 --- /dev/null +++ b/devops/ansible/default_flock/files/authorized_keys @@ -0,0 +1,2 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/ansible/default_flock/files/htpasswd b/devops/ansible/default_flock/files/htpasswd new file mode 100644 index 000000000..00db81b64 --- /dev/null +++ b/devops/ansible/default_flock/files/htpasswd @@ -0,0 +1 @@ +coglabs:$apr1$8DQw3IiE$wS.qtMUoAmfHhq3BQPQn30 diff --git a/devops/tasks/config/lo.sh b/devops/ansible/default_flock/files/lo.sh similarity index 100% rename from devops/tasks/config/lo.sh rename to devops/ansible/default_flock/files/lo.sh diff --git a/devops/ansible/default_flock/files/nginx b/devops/ansible/default_flock/files/nginx new file mode 100644 index 000000000..3067c6347 --- /dev/null +++ b/devops/ansible/default_flock/files/nginx @@ -0,0 +1,40 @@ +server { + # We listen for HTTP on port 80. When we set up certbot, this changes to 443. + listen 80 default_server; + listen [::]:80 default_server; + + server_name {{ hostname }}.{{ server_domain }}; + + location / { + # Generally, used to configure permissions. E.g. http basic auth, allow/deny + # IP blocks, etc. Note that for deploy, this should be broken out into several + # blocks (e.g. incoming event, dashboards, etc.) + {{nginx_root_options}} + + proxy_pass http://localhost:8888/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # We disable CORS globally. This should be more granular. + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + } + location /wsapi/ { + proxy_pass http://localhost:8888/wsapi/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 86400; + + add_header "Access-Control-Allow-Origin" *; + add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD"; + add_header "Access-Control-Allow-Headers" "Authorization, Origin, X-Requested-With, Content-Type, Accept"; + + if ($request_method = OPTIONS ) { + return 200; + } + } +} \ No newline at end of file diff --git a/devops/tasks/config/passwd.lo b/devops/ansible/default_flock/files/passwd.lo similarity index 100% rename from devops/tasks/config/passwd.lo rename to devops/ansible/default_flock/files/passwd.lo diff --git a/devops/tasks/config/systemd b/devops/ansible/default_flock/files/systemd similarity index 100% rename from devops/tasks/config/systemd rename to devops/ansible/default_flock/files/systemd diff --git a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML new file mode 100644 index 000000000..f9a519881 --- /dev/null +++ b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML @@ -0,0 +1,22 @@ + - name: Delete the symlink /etc/nginx/sites-enabled/default + file: + path: /etc/nginx/sites-enabled/default + state: absent + + - name: Delete the symlink /etc/nginx/sites-available/default + file: + path: /etc/nginx/sites-enabled/default + state: absent + + - name: Create a symlink for the nginx file + file: + src: /etc/nginx/sites-available/{{ inventory_hostname }} + dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} + state: link + + - name: Restart nginx + service: + name: nginx + state: restarted + become: yes + \ No newline at end of file diff --git a/devops/ansible/tasks/baseline.yaml b/devops/ansible/default_flock/yaml/BASELINE.YAML similarity index 90% rename from devops/ansible/tasks/baseline.yaml rename to devops/ansible/default_flock/yaml/BASELINE.YAML index 436e2c790..3f3b41758 100644 --- a/devops/ansible/tasks/baseline.yaml +++ b/devops/ansible/default_flock/yaml/BASELINE.YAML @@ -1,9 +1,3 @@ -- name: Provision writing analysis server - hosts: servers - become: yes - vars: - ansible_python_interpreter: /usr/bin/python3.12 - tasks: - apt: upgrade=dist update_cache=yes - name: Basic utils @@ -22,7 +16,7 @@ - wipe - build-essential - net-tools - + # We don't need all of this per se, but it's convenient. If nothing # else, it gives prereqs for `pip` - name: Install Python Packages @@ -66,7 +60,6 @@ - apache2-utils - fcgiwrap - python3-certbot-nginx - - virtualenvwrapper - name: Install pip apt: diff --git a/devops/ansible/default_flock/yaml/FILES.YAML b/devops/ansible/default_flock/yaml/FILES.YAML new file mode 100644 index 000000000..c585f5d9b --- /dev/null +++ b/devops/ansible/default_flock/yaml/FILES.YAML @@ -0,0 +1,12 @@ +files: + - name: nginx + dest: /etc/nginx/sites-available/{hostname} + - name: authorized_keys + dest: /home/ubuntu/authorized_keys + - name: passwd.lo + dest: /home/ubuntu/writing_observer/learning_observer/passwd.lo + - name: lo.sh + dest: /home/ubuntu/writing_observer/learning_observer/lo.sh + - name: systemd + dest: /etc/systemd/system/{hostname}.service + \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML b/devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML new file mode 100644 index 000000000..a5c2533db --- /dev/null +++ b/devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML @@ -0,0 +1,4 @@ + os_environment: + - key: "ANSIBLE_TEST" + value : "test_value" + \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/REPOS.YAML b/devops/ansible/default_flock/yaml/REPOS.YAML new file mode 100644 index 000000000..72792054b --- /dev/null +++ b/devops/ansible/default_flock/yaml/REPOS.YAML @@ -0,0 +1,5 @@ +repos: + - name: Writing Observer + repo: https://github.com/ETS-Next-Gen/writing_observer.git + dest: /home/ubuntu/writing_observer + branch: master \ No newline at end of file diff --git a/devops/ansible/default_flock/yamls.yaml b/devops/ansible/default_flock/yamls.yaml new file mode 100644 index 000000000..03ad42b78 --- /dev/null +++ b/devops/ansible/default_flock/yamls.yaml @@ -0,0 +1,7 @@ +yamls: + - name: ADDITIONAL_TASKS.YAML + - name: BASELINE.YAML + - name: FILES.YAML + - name: REPOS.YAML + - name: OS_ENV_VARS.YAML + \ No newline at end of file diff --git a/devops/ansible/files_to_upload/authorized_keys b/devops/ansible/files_to_upload/authorized_keys index b25f6c109..46d2e8ba5 100644 --- a/devops/ansible/files_to_upload/authorized_keys +++ b/devops/ansible/files_to_upload/authorized_keys @@ -1,4 +1,2 @@ -Copy the following into: ~/.ssh/authorized_keys - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCJ/ZWcQBJTXHHlmztnwY39sueZIR8CA5lKwBH9f7ra939sS/Gdgn1+OF/YUgFLC00GBtesuxMJB52xp2RaIxkrD56G1j64f3P1He22F8jdRUZ8+12PdhqtPsMD6eWofmyo5l2nNEwvQkwTQqQDRWQxuYBWmflHUKqyxPROSCCJXcq2gjVgvFcQZd8ZD24EKQAU3/T6lFwxeiOBNhD5EL6JQ/1Wc2kax9FNW+H8mVVPE+xhFQoGS/5phP2EoEA4EQ6Lpxc2AT68IsKPueKTK5vlpKl2qwt4u9UVT2aMeWapo/qXi1mJt8puBODkU28ygCK0LBQGr/SSbS+YRw16JXH5 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHmSeXMa7i6PxwTM/U/AHhVwSkWxni78hmxGs+MyZ28Ck6ndHRF5ex2JY/WZMHgJ5Z1ndOZ6riXmdex5gjjiKLpAYgyTR2Dl0aKo52+xIAK7OY6zs9WyV7XgOLDBzvoMewWFp3/2P88oVh2JEInzLe8hkl2npvW9/37ZL3+J9KAg9nQkFn6WPYH7hdJLqnmv25fabmZNsQmekXuz5jPuQbZ4n0zyLkbMxgmYPmgOlsHDO0Bomv9+gpffgWShkDwrXbasEg51OPZjbx/cssr383e0/dHGFPl0B+WRD7Pw796ehyFrE4HHC4nZYI2CS1iurfbPG8CtSUQM9xsAX7uZLl diff --git a/devops/ansible/hosts.ini b/devops/ansible/hosts.ini new file mode 100644 index 000000000..ce16bcb78 --- /dev/null +++ b/devops/ansible/hosts.ini @@ -0,0 +1,3 @@ +[servers] +brostpS2 ansible_host=3.84.57.3 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file diff --git a/devops/ansible/tasks/A_setup_flock.yaml b/devops/ansible/tasks/A_setup_flock.yaml new file mode 100644 index 000000000..a3d358e74 --- /dev/null +++ b/devops/ansible/tasks/A_setup_flock.yaml @@ -0,0 +1,72 @@ +- name: Setup flock for target_server if needed + hosts: localhost + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Read YAML files list + include_vars: + file: ../default_flock/yamls.yaml + + - name: Assign variables + set_fact: + dest_dir: "{{ flock_config }}/config/{{ target_server }}/yaml" + default_dir: "../default_flock/yaml" + file_list: "{{ yamls }}" + + - name: Ensure the destination directory exists + file: + path: "{{ dest_dir }}" + state: directory + mode: '0755' + + - name: Check if the file already exists on the server + stat: + path: "{{ dest_dir }}/{{ item.name }}" + register: file_stat + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Copy the file to the server if it does not exist + copy: + src: "{{ default_dir }}/{{ item.name }}" + dest: "{{ dest_dir }}/{{ item.name }}" + when: not (file_stat.results | selectattr('item.name', 'equalto', item.name) | map(attribute='stat.exists') | first) + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Read files list + include_vars: + file: ../default_flock/files.yaml + + - name: Assign variables + set_fact: + dest_dir: "{{ flock_config }}/config/{{ target_server }}/files" + default_dir: "../default_flock/files" + file_list: "{{ files }}" + + - name: Ensure the destination directory exists + file: + path: "{{ dest_dir }}" + state: directory + mode: '0755' + + - name: Check if the file already exists on the server + stat: + path: "{{ dest_dir }}/{{ item.name }}" + register: file_stat + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" + + - name: Copy the file to the server if it does not exist + copy: + src: "{{ default_dir }}/{{ item.name }}" + dest: "{{ dest_dir }}/{{ item.name }}" + when: not (file_stat.results | selectattr('item.name', 'equalto', item.name) | map(attribute='stat.exists') | first) + loop: "{{ file_list }}" + loop_control: + label: "{{ item.name }}" \ No newline at end of file diff --git a/devops/ansible/tasks/B_install_baseline.yaml b/devops/ansible/tasks/B_install_baseline.yaml new file mode 100644 index 000000000..933342b6b --- /dev/null +++ b/devops/ansible/tasks/B_install_baseline.yaml @@ -0,0 +1,12 @@ +- name: Provision writing analysis server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Include BASELINE.YAML from flock_config + include_tasks: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml/BASELINE.YAML" diff --git a/devops/ansible/tasks/install_repos.yaml b/devops/ansible/tasks/C_install_repos.yaml similarity index 63% rename from devops/ansible/tasks/install_repos.yaml rename to devops/ansible/tasks/C_install_repos.yaml index f4e243360..749d3d405 100644 --- a/devops/ansible/tasks/install_repos.yaml +++ b/devops/ansible/tasks/C_install_repos.yaml @@ -4,31 +4,19 @@ vars: ansible_python_interpreter: /usr/bin/python3.12 tasks: - - name: Install Git - apt: - name: git - state: present - - name: Read credentials include_vars: file: ../../tasks/settings/CREDS.YAML + - name: Assign variables + set_fact: + yaml_file: "REPOS.YAML" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + - name: Read repository configuration include_vars: - file: ../../tasks/settings/WO_REPOS.YAML - -# - name: Debug git_username variable -# debug: -# var: git_username - -# - name: Debug git_pac variable -# debug: -# var: git_pac - -# - name: Debug repos variable -# debug: -# var: repos - + file: "{{ yaml_dir }}/{{ yaml_file }}" + - name: Clone Git repositories git: repo: "https://{{ git_username }}:{{ git_pac }}@{{ item.repo | regex_replace('https://', '') }}" diff --git a/devops/ansible/tasks/test.yaml b/devops/ansible/tasks/D_copy_files.yaml similarity index 57% rename from devops/ansible/tasks/test.yaml rename to devops/ansible/tasks/D_copy_files.yaml index 718ef3043..daaf892a9 100644 --- a/devops/ansible/tasks/test.yaml +++ b/devops/ansible/tasks/D_copy_files.yaml @@ -3,11 +3,21 @@ become: yes vars: ansible_python_interpreter: /usr/bin/python3.12 - vars_files: - - ../../tasks/settings/CREDS.yaml - - ../../tasks/settings/TOY_SBA_FILES.yaml - tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "FILES.YAML" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + - name: Ensure destination directories exist file: path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" @@ -16,7 +26,7 @@ - name: Copy files to server template: - src: "{{ item.src }}" + src: "{{files_dir }}/{{ item.name }}" dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" owner: root group: root @@ -27,8 +37,4 @@ server_domain: "{{ domain }}" nginx_root_options: "" - - name: Restart nginx - service: - name: nginx - state: restarted - become: yes + \ No newline at end of file diff --git a/devops/ansible/tasks/os_environment_variables.yaml b/devops/ansible/tasks/E_setup_os_environment_variables.yaml similarity index 58% rename from devops/ansible/tasks/os_environment_variables.yaml rename to devops/ansible/tasks/E_setup_os_environment_variables.yaml index 04e13517a..71ca69dac 100644 --- a/devops/ansible/tasks/os_environment_variables.yaml +++ b/devops/ansible/tasks/E_setup_os_environment_variables.yaml @@ -4,10 +4,19 @@ vars: ansible_python_interpreter: /usr/bin/python3.12 tasks: - - name: Read environment configuration + - name: Read credentials include_vars: file: ../../tasks/settings/CREDS.YAML + - name: Assign variables + set_fact: + yaml_file: "OS_ENV_VARS.YAML" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + - name: Populate /etc/environment lineinfile: path: "/etc/environment" diff --git a/devops/ansible/tasks/F_run_additional_tasks.yaml b/devops/ansible/tasks/F_run_additional_tasks.yaml new file mode 100644 index 000000000..40ac6c625 --- /dev/null +++ b/devops/ansible/tasks/F_run_additional_tasks.yaml @@ -0,0 +1,17 @@ +- name: Run additional Ansible Tasks + hosts: all + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "ADDITIONAL_TASKS.YAML" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + + - name: Include additional tasks + include_tasks: "{{ yaml_dir }}/{{ yaml_file }}" \ No newline at end of file diff --git a/devops/ansible/tasks/G_download_config.yaml b/devops/ansible/tasks/G_download_config.yaml new file mode 100644 index 000000000..692705b54 --- /dev/null +++ b/devops/ansible/tasks/G_download_config.yaml @@ -0,0 +1,34 @@ +- name: Download files from server + hosts: servers + become: yes + vars: + ansible_python_interpreter: /usr/bin/python3.12 + tasks: + - name: Read credentials + include_vars: + file: ../../tasks/settings/CREDS.YAML + + - name: Assign variables + set_fact: + yaml_file: "FILES.YAML" + yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" + files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" + + - name: Read repository configuration + include_vars: + file: "{{ yaml_dir }}/{{ yaml_file }}" + + - name: Ensure the local files directory exists + file: + path: "{{ files_dir }}" + state: directory + mode: '0755' + + - name: download files from server + fetch: + src: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" + dest: "{{files_dir }}/{{ item.name }}" + flat: yes + loop: "{{ files }}" + + \ No newline at end of file diff --git a/devops/ansible/tasks/copy_files.yaml b/devops/ansible/tasks/copy_files.yaml deleted file mode 100644 index d5fdf1acf..000000000 --- a/devops/ansible/tasks/copy_files.yaml +++ /dev/null @@ -1,51 +0,0 @@ -- name: Copy files to server - hosts: servers - become: yes - vars: - ansible_python_interpreter: /usr/bin/python3.12 - vars_files: - - ../../tasks/settings/LO_FILES.yaml - - ../../tasks/settings/CREDS.yaml - - tasks: - - name: Ensure destination directories exist - file: - path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" - state: directory - loop: "{{ files }}" - - - name: Copy files to server - template: - src: "{{ item.src }}" - dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" - owner: root - group: root - mode: '0644' - loop: "{{ files }}" - vars: - hostname: "{{ inventory_hostname }}" - server_domain: "{{ domain }}" - nginx_root_options: "" - - - name: Delete the symlink /etc/nginx/sites-enabled/default - file: - path: /etc/nginx/sites-enabled/default - state: absent - - - name: Delete the symlink /etc/nginx/sites-available/default - file: - path: /etc/nginx/sites-enabled/default - state: absent - - - name: Create a symlink for the nginx file - file: - src: /etc/nginx/sites-available/{{ inventory_hostname }} - dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} - state: link - - - name: Restart nginx - service: - name: nginx - state: restarted - become: yes - \ No newline at end of file diff --git a/devops/ansible/tasks/master_playbook.yaml b/devops/ansible/tasks/master_playbook.yaml new file mode 100644 index 000000000..fc224c29b --- /dev/null +++ b/devops/ansible/tasks/master_playbook.yaml @@ -0,0 +1,14 @@ +- name: Install baseline packages + import_playbook: B_install_baseline.yaml + +- name: Install repos + import_playbook: C_install_repos.yaml + +- name: Copy files to server + import_playbook: D_copy_files.yaml + +- name: Setup os environment variables + import_playbook: E_setup_os_environment_variables.yaml + +- name: Run additional tasks for server + import_playbook: F_run_additional_tasks.yaml \ No newline at end of file diff --git a/devops/ansible/tasks/toy-sba.yaml b/devops/ansible/tasks/toy-sba.yaml deleted file mode 100644 index c925c3d34..000000000 --- a/devops/ansible/tasks/toy-sba.yaml +++ /dev/null @@ -1,108 +0,0 @@ -- name: Additional provisions for toy-sba server - hosts: servers - become: yes - vars: - ansible_python_interpreter: /usr/bin/python3.12 - vars_files: - - ../../tasks/settings/TOY_SBA_FILES.yaml - - ../../tasks/settings/CREDS.yaml - - ../../tasks/settings/TOY_REPOS.YAML - - tasks: - - name: Populate /etc/environment - lineinfile: - path: "/etc/environment" - state: present - regexp: "^{{ item.key }}=" - line: "{{ item.key }}={{ item.value }}" - with_items: "{{ os_environment }}" - - - name: Clone Git repositories - git: - repo: "https://{{ git_username }}:{{ git_pac }}@{{ item.repo | regex_replace('https://', '') }}" - dest: "{{ item.dest }}" - clone: yes - update: yes - version: "{{ item.branch | default('master') }}" - force: yes - loop: "{{ repos }}" - - - name: Install npm - apt: - name: npm - state: present - - - name: Install pm2 using npm - npm: - name: pm2 - global: yes - - - name: Run npm install for lo_event - shell: npm install - args: - chdir: /home/ubuntu/writing_observer/modules/lo_event - - - name: Run npm install for toy-sba - shell: npm install --legacy-peer-deps - args: - chdir: /home/ubuntu/toy-sba - - - name: Ensure destination directories exist - file: - path: "{{ item.dest | dirname | replace('{hostname}', inventory_hostname) }}" - state: directory - loop: "{{ files }}" - - - name: Copy files to server - template: - src: "{{ item.src }}" - dest: "{{ item.dest | replace('{hostname}', inventory_hostname) }}" - owner: root - group: root - mode: '0644' - loop: "{{ files }}" - vars: - hostname: "{{ inventory_hostname }}" - server_domain: "{{ domain }}" - nginx_root_options: "" - - - name: Create a symlink for the nginx file - file: - src: /etc/nginx/sites-available/{ inventory_hostname }} - dest: /etc/nginx/sites-enabled/{ inventory_hostname }} - state: link - - - name: Restart nginx - service: - name: nginx - state: restarted - become: yes - - - name: Check if toy-sba is running with pm2 - shell: pm2 describe toy-sba - register: pm2_toy_sba_running - ignore_errors: true - failed_when: false - - - name: Debug pm2_toy_sba_running - debug: - msg: "pm2_toy_sba_running.rc is {{ pm2_toy_sba_running.rc }}" - - - name: Start toy-sba with pm2 if not running - shell: pm2 start npm --name "toy-sba" -- run dev - when: pm2_toy_sba_running.rc != 0 - args: - chdir: /home/ubuntu/toy-sba - become: yes - - - name: Wait for 5 seconds before restarting toy-sba - wait_for: - timeout: 5 - when: pm2_toy_sba_running.rc == 0 - - - name: Restart toy-sba with pm2 if running - shell: pm2 restart toy-sba - when: pm2_toy_sba_running.rc == 0 - args: - chdir: /home/ubuntu/toy-sba - become: yes diff --git a/devops/ansible/tasks/writing-apt.yaml b/devops/ansible/tasks/writing-apt.yaml deleted file mode 100644 index 5032f1234..000000000 --- a/devops/ansible/tasks/writing-apt.yaml +++ /dev/null @@ -1,74 +0,0 @@ -- apt: upgrade=dist update_cache=yes - -- name: Basic utils - apt: name={{ item }} - with_items: - - curl - - emacs - - git - - git-core - - links - - lynx - - mosh - - nmap - - whois - - screen - - wipe - - build-essential - - net-tools - -# We don't need all of this per se, but it's convenient. If nothing -# else, it gives prereqs for `pip` -- name: Install Python Packages - apt: - name: "{{ item }} " - state: present - with_items: - - ipython3 - - libxml2-dev - - libxslt1-dev - - python3-boto - - python3-bson - - python3-dev - - python3-matplotlib - - python3-numpy - - python3-pandas - - python3-pip - - python3-scipy - - python3-setuptools - - python3-sklearn - - virtualenvwrapper - - libjpeg-dev - - python3-opencv - - python3-virtualenv - - python3-aiohttp - - python3-aiohttp-cors - - python3-tornado - - python3-venv - - python3-yaml - - python3-asyncpg - - python3-bcrypt - -- name: Install server packages - apt: - name: "{{ item }}" - state: present - loop: - - redis - - nginx - - certbot - - apache2-utils - - fcgiwrap - - python3-certbot-nginx - - virtualenvwrapper - -- name: Install pip - apt: - name: python3-pip - state: present - -- name: Create a virtual environment for AWS CLI - command: python3 -m venv /opt/learning-observer - -- name: Install AWS CLI using pip in virtual environment - command: /opt/learning-observer/bin/pip install awscli \ No newline at end of file diff --git a/devops/tasks/README.md b/devops/tasks/README.md index 37a0e83e8..91861c76a 100644 --- a/devops/tasks/README.md +++ b/devops/tasks/README.md @@ -148,5 +148,7 @@ design: inv downloadconfig [machine] - inv certbot coglabs - inv downloadconfig coglabs \ No newline at end of file +###TODO- replace hosts.ini with a python script that returns inv list output in correct format + + + ansible-playbook -i hosts.ini ../ansible/tasks/baseline.yaml --limit coglabs \ No newline at end of file diff --git a/devops/tasks/config/creds.yaml b/devops/tasks/config/creds.yaml deleted file mode 100644 index dab0fe6d8..000000000 --- a/devops/tasks/config/creds.yaml +++ /dev/null @@ -1,33 +0,0 @@ -hostname: {{hostname}}.{{domain}} -xmpp: - sink: # Receives messages. We'll need many of these. - jid: sink@localhost - password: {{RANDOM1}} - source: # Sends messages. - jid: source@localhost - password: {{RANDOM1}} - stream: # For debugging - jid: stream@localhost - password: {{RANDOM1}} -auth: - password_file: passwd.lo -pubsub: - type: redis -kvs: - type: redis -roster_data: - source: all -aio: - session_secret: {{RANDOM2}} - session_max_age: 3600 -config: - run_mode: dev - debug: [] -theme: - server_name: Learning Observer - front_page_pitch: Learning Observer is an experimental dashboard. If you'd like to be part of the experiment, please contact us. If you're already part of the experiment, log in! - logo_big: /static/media/logo-clean.jpg -event_auth: - local_storage: - userfile: students.yaml - allow_guest: true \ No newline at end of file diff --git a/devops/tasks/config/authorized_keys b/devops/tasks/config_NO_LONGER_NEEDED/authorized_keys similarity index 100% rename from devops/tasks/config/authorized_keys rename to devops/tasks/config_NO_LONGER_NEEDED/authorized_keys diff --git a/devops/tasks/config/hostname b/devops/tasks/config_NO_LONGER_NEEDED/hostname similarity index 100% rename from devops/tasks/config/hostname rename to devops/tasks/config_NO_LONGER_NEEDED/hostname diff --git a/devops/tasks/config/init.d b/devops/tasks/config_NO_LONGER_NEEDED/init.d similarity index 100% rename from devops/tasks/config/init.d rename to devops/tasks/config_NO_LONGER_NEEDED/init.d diff --git a/devops/tasks/config_NO_LONGER_NEEDED/lo.sh b/devops/tasks/config_NO_LONGER_NEEDED/lo.sh new file mode 100644 index 000000000..d9a05d850 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/lo.sh @@ -0,0 +1,9 @@ +#!/usr/bin/bash + +# This is a script to start up Learning Observer with it's own process +# name. This is convenient for being able to start / stop the process. + +. /usr/share/virtualenvwrapper/virtualenvwrapper.sh +workon learning_observer +cd /home/ubuntu/writing_observer/learning_observer +bash -c "exec -a learning_observer python learning_observer" >> /home/ubuntu/lo.log 2>> /home/ubuntu/lo.err diff --git a/devops/tasks/config/nginx b/devops/tasks/config_NO_LONGER_NEEDED/nginx similarity index 100% rename from devops/tasks/config/nginx rename to devops/tasks/config_NO_LONGER_NEEDED/nginx diff --git a/devops/tasks/config/nginx_toy_sba b/devops/tasks/config_NO_LONGER_NEEDED/nginx_toy_sba similarity index 100% rename from devops/tasks/config/nginx_toy_sba rename to devops/tasks/config_NO_LONGER_NEEDED/nginx_toy_sba diff --git a/devops/tasks/config_NO_LONGER_NEEDED/passwd.lo b/devops/tasks/config_NO_LONGER_NEEDED/passwd.lo new file mode 100644 index 000000000..e69de29bb diff --git a/devops/tasks/config/postuploads b/devops/tasks/config_NO_LONGER_NEEDED/postuploads similarity index 100% rename from devops/tasks/config/postuploads rename to devops/tasks/config_NO_LONGER_NEEDED/postuploads diff --git a/devops/tasks/config/rsyslog.conf b/devops/tasks/config_NO_LONGER_NEEDED/rsyslog.conf similarity index 100% rename from devops/tasks/config/rsyslog.conf rename to devops/tasks/config_NO_LONGER_NEEDED/rsyslog.conf diff --git a/devops/tasks/config/sync.csv b/devops/tasks/config_NO_LONGER_NEEDED/sync.csv similarity index 100% rename from devops/tasks/config/sync.csv rename to devops/tasks/config_NO_LONGER_NEEDED/sync.csv diff --git a/devops/tasks/config_NO_LONGER_NEEDED/systemd b/devops/tasks/config_NO_LONGER_NEEDED/systemd new file mode 100644 index 000000000..673d3ffa2 --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/systemd @@ -0,0 +1,11 @@ +[Unit] +Description=Learning Observer + +[Service] +ExecStart=/home/ubuntu/writing_observer/learning_observer/lo.sh +Type=simple +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=learning_observer +User=ubuntu +Group=ubuntu \ No newline at end of file diff --git a/devops/tasks/config/systemd_toy_sba b/devops/tasks/config_NO_LONGER_NEEDED/systemd_toy_sba similarity index 100% rename from devops/tasks/config/systemd_toy_sba rename to devops/tasks/config_NO_LONGER_NEEDED/systemd_toy_sba diff --git a/devops/tasks/config/toy-sba.sh b/devops/tasks/config_NO_LONGER_NEEDED/toy-sba.sh similarity index 100% rename from devops/tasks/config/toy-sba.sh rename to devops/tasks/config_NO_LONGER_NEEDED/toy-sba.sh diff --git a/devops/tasks/hosts.ini b/devops/tasks/hosts.ini index ce16bcb78..e161b21a0 100644 --- a/devops/tasks/hosts.ini +++ b/devops/tasks/hosts.ini @@ -1,3 +1,4 @@ [servers] brostpS2 ansible_host=3.84.57.3 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem -coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file +coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +daclassroom ansible_host=18.207.219.42 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file diff --git a/devops/tasks/orchlib/aws.py b/devops/tasks/orchlib/aws.py index a6adad5f8..b37f76cae 100644 --- a/devops/tasks/orchlib/aws.py +++ b/devops/tasks/orchlib/aws.py @@ -48,8 +48,8 @@ def create_instance(name): 'Value': orchlib.config.creds['owner'] }, { - 'Key': 'deploy-group', - 'Value': orchlib.config.creds['deploy-group'] + 'Key': 'deploy_group', + 'Value': orchlib.config.creds['deploy_group'] }, { 'Key': 'Patch Group', @@ -127,8 +127,8 @@ def list_instances(): ''' reservations = ec2client.describe_instances(Filters=[ { - 'Name': 'tag:deploy-group', - 'Values': [orchlib.config.creds['deploy-group']] + 'Name': 'tag:deploy_group', + 'Values': [orchlib.config.creds['deploy_group']] }, ])['Reservations'] instances = sum([i['Instances'] for i in reservations], []) diff --git a/devops/tasks/orchlib/config.py b/devops/tasks/orchlib/config.py index 84233b1db..dcc3205cf 100644 --- a/devops/tasks/orchlib/config.py +++ b/devops/tasks/orchlib/config.py @@ -23,8 +23,8 @@ "openai_deployment_id": "OPEN AI deployment ID", "openai_api_key": "your api key for the azure instance", "domain": "Domain name (e.g. learning-observer.org)", - "flock-config": "Path to git repo where we'll store machine config.", - "deploy-group": "Tag to identify all machines (typically, learning-observer)", + "flock_config": "Path to git repo where we'll store machine config.", + "deploy_group": "Tag to identify all machines (typically, learning-observer)", "ec2_tags": "JSON dictionary of any additional tags you'd like on your machines. If you're not sure, type {}" } print("I'll need:") @@ -36,9 +36,9 @@ print(value) d[key] = input("{key}: ".format(key=key)).strip() d['ec2_tags'] = json.loads(d['ec2_tags']) - if not os.path.exists(d['flock-config']): - os.system("git init {path}".format(path=d['flock-config'])) - os.mkdir(os.path.join(d['flock-config'], "config")) + if not os.path.exists(d['flock_config']): + os.system("git init {path}".format(path=d['flock_config'])) + os.mkdir(os.path.join(d['flock_config'], "config")) with open("settings/CREDS.YAML", "w") as fp: yaml.dump(d, fp) @@ -59,15 +59,15 @@ def config_filename(machine_name, file_suffix, create=False): paths = [ # First, we try per-machine configuration os.path.join( - creds["flock-config"], "config", machine_name, file_suffix + creds["flock_config"], "config", machine_name, file_suffix ), # Next, we try the per-machine override os.path.join( - creds["flock-config"], "config", machine_name, file_suffix+".base" + creds["flock_config"], "config", machine_name, file_suffix+".base" ), # Then, system-wide configuration os.path.join( - creds["flock-config"], "config", file_suffix + creds["flock_config"], "config", file_suffix ), # And finally, as a fallback, default files os.path.join( diff --git a/devops/tasks/orchlib/logger.py b/devops/tasks/orchlib/logger.py index 2622b4d62..667054121 100644 --- a/devops/tasks/orchlib/logger.py +++ b/devops/tasks/orchlib/logger.py @@ -37,5 +37,5 @@ def exitlog(): Not done. ''' os.path.join( - orchlib.config.creds["flock-config"], "logs" + orchlib.config.creds["flock_config"], "logs" ) diff --git a/devops/tasks/tasks.py b/devops/tasks/tasks.py index 33e34e333..069dd6f29 100644 --- a/devops/tasks/tasks.py +++ b/devops/tasks/tasks.py @@ -71,8 +71,7 @@ def initialize(c, machine_name): print("DNS....") orchlib.aws.register_dns(machine_name, orchlib.config.creds['domain'], ip) print("IP", ip) - orchlib.ubuntu.update(ip) - + # Write to hosts.ini hosts_ini_path = '../settings/hosts.ini' line_to_write = f"{machine_name} ansible_host={ip} ansible_user=ubuntu ansible_ssh_private_key_file={orchlib.config.creds['key_filename']}\n" @@ -367,7 +366,7 @@ def commit(c, msg): ''' system( "cd {gitpath} ; git add -A; git commit -m {msg}".format( - gitpath=orchlib.config.creds["flock-config"], + gitpath=orchlib.config.creds["flock_config"], msg=msg ) ) From de973923ae0fffda92c73243f810c9024f2de1ac Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Thu, 26 Sep 2024 23:27:03 -0400 Subject: [PATCH 06/16] file case syntax fixed, added cred.yaml to deployment files, added additional tasks to do some of env fab script --- devops/ansible/default_flock/files.yaml | 4 ++ .../default_flock/yaml/ADDITIONAL_TASKS.YAML | 64 +++++++++++++------ devops/ansible/default_flock/yamls.yaml | 10 +-- devops/ansible/tasks/B_install_baseline.yaml | 2 +- devops/ansible/tasks/C_install_repos.yaml | 2 +- devops/ansible/tasks/D_copy_files.yaml | 7 +- .../E_setup_os_environment_variables.yaml | 2 +- .../ansible/tasks/F_run_additional_tasks.yaml | 2 +- devops/ansible/tasks/G_download_config.yaml | 2 +- 9 files changed, 64 insertions(+), 31 deletions(-) diff --git a/devops/ansible/default_flock/files.yaml b/devops/ansible/default_flock/files.yaml index 42627acb2..ac024b996 100644 --- a/devops/ansible/default_flock/files.yaml +++ b/devops/ansible/default_flock/files.yaml @@ -2,4 +2,8 @@ files: - name: authorized_keys - name: htpasswd - name: nginx + - name: creds.yaml + - name: lo.sh + - name: password.lo + - name: systemd \ No newline at end of file diff --git a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML index f9a519881..6d52a3224 100644 --- a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML +++ b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML @@ -1,22 +1,46 @@ - - name: Delete the symlink /etc/nginx/sites-enabled/default - file: - path: /etc/nginx/sites-enabled/default - state: absent - - - name: Delete the symlink /etc/nginx/sites-available/default - file: - path: /etc/nginx/sites-enabled/default - state: absent +- name: Change ownership to ubuntu:ubuntu for /home/ubuntu + ansible.builtin.file: + path: /home/ubuntu + state: directory + recurse: yes + owner: ubuntu + group: ubuntu - - name: Create a symlink for the nginx file - file: - src: /etc/nginx/sites-available/{{ inventory_hostname }} - dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} - state: link +- name: Delete the symlink /etc/nginx/sites-enabled/default + ansible.builtin.file: + path: /etc/nginx/sites-enabled/default + state: absent - - name: Restart nginx - service: - name: nginx - state: restarted - become: yes - \ No newline at end of file +- name: Delete the symlink /etc/nginx/sites-available/default + ansible.builtin.file: + path: /etc/nginx/sites-available/default + state: absent + +- name: Create a symlink for the nginx file + ansible.builtin.file: + src: /etc/nginx/sites-available/{{ inventory_hostname }} + dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} + state: link + +- name: Create a virtual environment named learning_observer + ansible.builtin.command: + cmd: python3 -m venv /home/ubuntu/.virtualenvs/learning_observer + args: + creates: /home/ubuntu/.virtualenvs/learning_observer/bin/activate + +- name: Install requirements from requirements.txt in virtual environment + ansible.builtin.command: + cmd: /home/ubuntu/.virtualenvs/learning_observer/bin/pip install -r requirements.txt + chdir: /home/ubuntu/writing_observer + +- name: Install learning observer in virtual environment + ansible.builtin.command: + cmd: /home/ubuntu/.virtualenvs/learning_observer/bin/pip install -e learning_observer + chdir: /home/ubuntu/writing_observer + +- name: Restart nginx + ansible.builtin.service: + name: nginx + state: restarted + become: yes + \ No newline at end of file diff --git a/devops/ansible/default_flock/yamls.yaml b/devops/ansible/default_flock/yamls.yaml index 03ad42b78..88456e536 100644 --- a/devops/ansible/default_flock/yamls.yaml +++ b/devops/ansible/default_flock/yamls.yaml @@ -1,7 +1,7 @@ yamls: - - name: ADDITIONAL_TASKS.YAML - - name: BASELINE.YAML - - name: FILES.YAML - - name: REPOS.YAML - - name: OS_ENV_VARS.YAML + - name: additional_tasks.yaml + - name: baseline.yaml + - name: files.yaml + - name: repos.yaml + - name: os_env_vars.yaml \ No newline at end of file diff --git a/devops/ansible/tasks/B_install_baseline.yaml b/devops/ansible/tasks/B_install_baseline.yaml index 933342b6b..37a34f35b 100644 --- a/devops/ansible/tasks/B_install_baseline.yaml +++ b/devops/ansible/tasks/B_install_baseline.yaml @@ -9,4 +9,4 @@ file: ../../tasks/settings/CREDS.YAML - name: Include BASELINE.YAML from flock_config - include_tasks: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml/BASELINE.YAML" + include_tasks: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml/baseline.yaml" diff --git a/devops/ansible/tasks/C_install_repos.yaml b/devops/ansible/tasks/C_install_repos.yaml index 749d3d405..94e904512 100644 --- a/devops/ansible/tasks/C_install_repos.yaml +++ b/devops/ansible/tasks/C_install_repos.yaml @@ -10,7 +10,7 @@ - name: Assign variables set_fact: - yaml_file: "REPOS.YAML" + yaml_file: "repos.yaml" yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" - name: Read repository configuration diff --git a/devops/ansible/tasks/D_copy_files.yaml b/devops/ansible/tasks/D_copy_files.yaml index daaf892a9..7ab706f43 100644 --- a/devops/ansible/tasks/D_copy_files.yaml +++ b/devops/ansible/tasks/D_copy_files.yaml @@ -10,7 +10,7 @@ - name: Assign variables set_fact: - yaml_file: "FILES.YAML" + yaml_file: "files.yaml" yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" @@ -24,6 +24,10 @@ state: directory loop: "{{ files }}" + - name: Generate a random number + set_fact: + random_num: "{{ 1000000 | random }}" + - name: Copy files to server template: src: "{{files_dir }}/{{ item.name }}" @@ -36,5 +40,6 @@ hostname: "{{ inventory_hostname }}" server_domain: "{{ domain }}" nginx_root_options: "" + RANDOM1: "{{ random_num }}" \ No newline at end of file diff --git a/devops/ansible/tasks/E_setup_os_environment_variables.yaml b/devops/ansible/tasks/E_setup_os_environment_variables.yaml index 71ca69dac..c837846e3 100644 --- a/devops/ansible/tasks/E_setup_os_environment_variables.yaml +++ b/devops/ansible/tasks/E_setup_os_environment_variables.yaml @@ -10,7 +10,7 @@ - name: Assign variables set_fact: - yaml_file: "OS_ENV_VARS.YAML" + yaml_file: "os_env_vars.yaml" yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" - name: Read repository configuration diff --git a/devops/ansible/tasks/F_run_additional_tasks.yaml b/devops/ansible/tasks/F_run_additional_tasks.yaml index 40ac6c625..0c5f5e0d3 100644 --- a/devops/ansible/tasks/F_run_additional_tasks.yaml +++ b/devops/ansible/tasks/F_run_additional_tasks.yaml @@ -10,7 +10,7 @@ - name: Assign variables set_fact: - yaml_file: "ADDITIONAL_TASKS.YAML" + yaml_file: "additional_tasks.yaml" yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" - name: Include additional tasks diff --git a/devops/ansible/tasks/G_download_config.yaml b/devops/ansible/tasks/G_download_config.yaml index 692705b54..c88209111 100644 --- a/devops/ansible/tasks/G_download_config.yaml +++ b/devops/ansible/tasks/G_download_config.yaml @@ -10,7 +10,7 @@ - name: Assign variables set_fact: - yaml_file: "FILES.YAML" + yaml_file: "files.yaml" yaml_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/yaml" files_dir: "{{ flock_config }}/config/{{ inventory_hostname }}/files" From 817e36f254dfb621af35c7a341abbb17c0760593 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 08:48:18 -0400 Subject: [PATCH 07/16] checking in latest. Still getting an issue starting LO --- .../default_flock/yaml/ADDITIONAL_TASKS.YAML | 39 ++++++++++++------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML index 6d52a3224..805e9cb11 100644 --- a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML +++ b/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML @@ -22,21 +22,34 @@ dest: /etc/nginx/sites-enabled/{{ inventory_hostname }} state: link -- name: Create a virtual environment named learning_observer - ansible.builtin.command: - cmd: python3 -m venv /home/ubuntu/.virtualenvs/learning_observer - args: - creates: /home/ubuntu/.virtualenvs/learning_observer/bin/activate +- name: Ensure virtualenvwrapper is installed + ansible.builtin.package: + name: virtualenvwrapper + state: present -- name: Install requirements from requirements.txt in virtual environment - ansible.builtin.command: - cmd: /home/ubuntu/.virtualenvs/learning_observer/bin/pip install -r requirements.txt - chdir: /home/ubuntu/writing_observer +- name: Add virtualenvwrapper initialization to profile + ansible.builtin.shell: | + echo ". /usr/share/virtualenvwrapper/virtualenvwrapper.sh" >> /home/ubuntu/.profile + args: + executable: /bin/bash -- name: Install learning observer in virtual environment - ansible.builtin.command: - cmd: /home/ubuntu/.virtualenvs/learning_observer/bin/pip install -e learning_observer - chdir: /home/ubuntu/writing_observer +- name: Source profile and create virtual environment learning_observer + ansible.builtin.shell: | + cd /home/ubuntu/writing_observer + make install + . /home/ubuntu/.profile + mkvirtualenv learning_observer + echo "workon learning_observer" >> /home/ubuntu/.profile + . /home/ubuntu/.profile + pip install --upgrade pip + cd /home/ubuntu/writing_observer/ + pip install -r requirements.txt + cd /home/ubuntu/writing_observer/learning_observer/ + python setup.py develop + args: + executable: /bin/bash + become: yes + become_user: ubuntu - name: Restart nginx ansible.builtin.service: From b466a1b4b9f04379d7b4aeb832ac3c4b83100f46 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 10:01:01 -0400 Subject: [PATCH 08/16] change to systemd --- devops/ansible/default_flock/files/systemd | 5 ++++- devops/ansible/default_flock/yaml/FILES.YAML | 2 +- devops/ansible/tasks/D_copy_files.yaml | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/devops/ansible/default_flock/files/systemd b/devops/ansible/default_flock/files/systemd index 673d3ffa2..a789e4513 100644 --- a/devops/ansible/default_flock/files/systemd +++ b/devops/ansible/default_flock/files/systemd @@ -8,4 +8,7 @@ StandardOutput=syslog StandardError=syslog SyslogIdentifier=learning_observer User=ubuntu -Group=ubuntu \ No newline at end of file +Group=ubuntu + +[Install] +WantedBy=multi-user.target diff --git a/devops/ansible/default_flock/yaml/FILES.YAML b/devops/ansible/default_flock/yaml/FILES.YAML index c585f5d9b..2cb6d9473 100644 --- a/devops/ansible/default_flock/yaml/FILES.YAML +++ b/devops/ansible/default_flock/yaml/FILES.YAML @@ -2,7 +2,7 @@ files: - name: nginx dest: /etc/nginx/sites-available/{hostname} - name: authorized_keys - dest: /home/ubuntu/authorized_keys + dest: /home/ubuntu/.ssh/authorized_keys - name: passwd.lo dest: /home/ubuntu/writing_observer/learning_observer/passwd.lo - name: lo.sh diff --git a/devops/ansible/tasks/D_copy_files.yaml b/devops/ansible/tasks/D_copy_files.yaml index 7ab706f43..40c754026 100644 --- a/devops/ansible/tasks/D_copy_files.yaml +++ b/devops/ansible/tasks/D_copy_files.yaml @@ -28,6 +28,10 @@ set_fact: random_num: "{{ 1000000 | random }}" + - name: Generate a random GUID + set_fact: + random_guid: "{{ lookup('pipe', 'uuidgen') }}" + - name: Copy files to server template: src: "{{files_dir }}/{{ item.name }}" @@ -40,6 +44,6 @@ hostname: "{{ inventory_hostname }}" server_domain: "{{ domain }}" nginx_root_options: "" - RANDOM1: "{{ random_num }}" + RANDOM1: "{{ random_guid }}" \ No newline at end of file From 9c0b7d5c2768c637b431fb796bda31bd0056256a Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 10:13:16 -0400 Subject: [PATCH 09/16] typo in default flock file.yaml --- devops/ansible/default_flock/files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/ansible/default_flock/files.yaml b/devops/ansible/default_flock/files.yaml index ac024b996..7de817dec 100644 --- a/devops/ansible/default_flock/files.yaml +++ b/devops/ansible/default_flock/files.yaml @@ -4,6 +4,6 @@ files: - name: nginx - name: creds.yaml - name: lo.sh - - name: password.lo + - name: passwrd.lo - name: systemd \ No newline at end of file From 5033fb4da9767572022bd5214e97cb5e690acb39 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 10:13:57 -0400 Subject: [PATCH 10/16] typo in default flock file.yaml --- devops/ansible/default_flock/files.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/ansible/default_flock/files.yaml b/devops/ansible/default_flock/files.yaml index 7de817dec..7bc82b71c 100644 --- a/devops/ansible/default_flock/files.yaml +++ b/devops/ansible/default_flock/files.yaml @@ -4,6 +4,6 @@ files: - name: nginx - name: creds.yaml - name: lo.sh - - name: passwrd.lo + - name: passwd.lo - name: systemd \ No newline at end of file From 1575b6bb86b38c12bfcf68569e79c38570e766d2 Mon Sep 17 00:00:00 2001 From: Piotr Mitros Date: Mon, 30 Sep 2024 10:24:57 -0400 Subject: [PATCH 11/16] Fix case issue --- .../yaml/{ADDITIONAL_TASKS.YAML => additional_tasks.yaml} | 0 .../ansible/default_flock/yaml/{BASELINE.YAML => baseline.yaml} | 0 devops/ansible/default_flock/yaml/{FILES.YAML => files.yaml} | 0 .../default_flock/yaml/{OS_ENV_VARS.YAML => os_env_vars.yaml} | 0 devops/ansible/default_flock/yaml/{REPOS.YAML => repos.yaml} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename devops/ansible/default_flock/yaml/{ADDITIONAL_TASKS.YAML => additional_tasks.yaml} (100%) rename devops/ansible/default_flock/yaml/{BASELINE.YAML => baseline.yaml} (100%) rename devops/ansible/default_flock/yaml/{FILES.YAML => files.yaml} (100%) rename devops/ansible/default_flock/yaml/{OS_ENV_VARS.YAML => os_env_vars.yaml} (100%) rename devops/ansible/default_flock/yaml/{REPOS.YAML => repos.yaml} (100%) diff --git a/devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML b/devops/ansible/default_flock/yaml/additional_tasks.yaml similarity index 100% rename from devops/ansible/default_flock/yaml/ADDITIONAL_TASKS.YAML rename to devops/ansible/default_flock/yaml/additional_tasks.yaml diff --git a/devops/ansible/default_flock/yaml/BASELINE.YAML b/devops/ansible/default_flock/yaml/baseline.yaml similarity index 100% rename from devops/ansible/default_flock/yaml/BASELINE.YAML rename to devops/ansible/default_flock/yaml/baseline.yaml diff --git a/devops/ansible/default_flock/yaml/FILES.YAML b/devops/ansible/default_flock/yaml/files.yaml similarity index 100% rename from devops/ansible/default_flock/yaml/FILES.YAML rename to devops/ansible/default_flock/yaml/files.yaml diff --git a/devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML b/devops/ansible/default_flock/yaml/os_env_vars.yaml similarity index 100% rename from devops/ansible/default_flock/yaml/OS_ENV_VARS.YAML rename to devops/ansible/default_flock/yaml/os_env_vars.yaml diff --git a/devops/ansible/default_flock/yaml/REPOS.YAML b/devops/ansible/default_flock/yaml/repos.yaml similarity index 100% rename from devops/ansible/default_flock/yaml/REPOS.YAML rename to devops/ansible/default_flock/yaml/repos.yaml From 1cf9a250074a3788595df5d5c75b3e491ec72145 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 10:31:50 -0400 Subject: [PATCH 12/16] added default flock creds.yaml --- .gitignore | 5 ++- devops/ansible/default_flock/files/creds.yaml | 37 +++++++++++++++++++ .../tasks/config_NO_LONGER_NEEDED/creds.yaml | 23 ++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 devops/ansible/default_flock/files/creds.yaml create mode 100644 devops/tasks/config_NO_LONGER_NEEDED/creds.yaml diff --git a/.gitignore b/.gitignore index d951ebd57..032b07a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,8 +4,9 @@ *__pycache__* webapp/logs webapp/static_data/teachers.yaml -creds.yaml -CREDS.YAML +learning_observer/*/creds.yaml +learning_observer/*/CREDS.YAML +devops/tasks/settings/CREDS.YAML uncommitted extension.crx extension.pem diff --git a/devops/ansible/default_flock/files/creds.yaml b/devops/ansible/default_flock/files/creds.yaml new file mode 100644 index 000000000..ce1379199 --- /dev/null +++ b/devops/ansible/default_flock/files/creds.yaml @@ -0,0 +1,37 @@ +hostname: {{hostname}}.{{server_domain}} +auth: + password_file: passwd.lo +pubsub: + type: redis +kvs: + # stub for in-memory debugging + # redis_ephemeral for redis / debugging (object disappear) + # redis for deploy + # If using redis_ephemeral, persist objects for 60s = 1 minute + # I typically use: + # * 1-10s for test cases + # * 1-5 minutes for interactive debugging + # * 6-24 hours for development + default: + type: stub + expiry: 6000 + memoization: + type: redis_ephemeral + expiry: 60 +feature_flags: {} +roster_data: + source: all +aio: + session_secret: {{RANDOM1}} + session_max_age: 3600 +config: + run_mode: dev + debug: [] +theme: + server_name: Learning Observer + front_page_pitch: Learning Observer is an experimental dashboard. If you'd like to be part of the experiment, please contact us. If you're already part of the experiment, log in! + logo_big: /static/media/logo-clean.jpg +event_auth: + local_storage: + userfile: students.yaml + allow_guest: true \ No newline at end of file diff --git a/devops/tasks/config_NO_LONGER_NEEDED/creds.yaml b/devops/tasks/config_NO_LONGER_NEEDED/creds.yaml new file mode 100644 index 000000000..b626b36de --- /dev/null +++ b/devops/tasks/config_NO_LONGER_NEEDED/creds.yaml @@ -0,0 +1,23 @@ +hostname: {{hostname}}.{{domain}} +auth: + password_file: passwd.lo +pubsub: + type: redis +kvs: + type: redis +roster_data: + source: all +aio: + session_secret: {{RANDOM2}} + session_max_age: 3600 +config: + run_mode: dev + debug: [] +theme: + server_name: Learning Observer + front_page_pitch: Learning Observer is an experimental dashboard. If you'd like to be part of the experiment, please contact us. If you're already part of the experiment, log in! + logo_big: /static/media/logo-clean.jpg +event_auth: + local_storage: + userfile: students.yaml + allow_guest: true \ No newline at end of file From 6bcaa8e3a7dd9865892d5b237edb0c5ed8e1c827 Mon Sep 17 00:00:00 2001 From: Paul Brost Date: Mon, 30 Sep 2024 14:57:23 -0400 Subject: [PATCH 13/16] python script to update hosts --- devops/tasks/hosts.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 devops/tasks/hosts.py diff --git a/devops/tasks/hosts.py b/devops/tasks/hosts.py new file mode 100644 index 000000000..e33708360 --- /dev/null +++ b/devops/tasks/hosts.py @@ -0,0 +1,18 @@ +import orchlib.aws + +def update_hosts_ini(): + # Fetch list of instances + instances = orchlib.aws.list_instances() + + # Open or create the hosts.ini file + with open('hosts.ini', 'w') as hosts_file: + # Write a default header or group for the ansible inventory + hosts_file.write('[aws_servers]\n') + + # Iterate through the instances and write their IP addresses + for instance in instances: + # Assuming the instance object has an attribute 'ip_address' + hosts_file.write(f"{instance.ip_address}\n") + +# Call the function to update the hosts.ini file +update_hosts_ini() \ No newline at end of file From d9e269d8f71de2b3cd4953bf5cffa70a45734550 Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 15:25:02 -0400 Subject: [PATCH 14/16] fixed hosts.py --- devops/tasks/hosts.ini | 8 +++++--- devops/tasks/hosts.py | 20 ++++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/devops/tasks/hosts.ini b/devops/tasks/hosts.ini index e161b21a0..a27d50cec 100644 --- a/devops/tasks/hosts.ini +++ b/devops/tasks/hosts.ini @@ -1,4 +1,6 @@ -[servers] -brostpS2 ansible_host=3.84.57.3 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +[aws_servers] +open-house ansible_host=3.87.42.157 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +da-i18n ansible_host=54.166.118.68 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem +da ansible_host=44.202.219.209 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem coglabs ansible_host=44.201.217.150 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem -daclassroom ansible_host=18.207.219.42 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem \ No newline at end of file +daclassroom ansible_host=18.207.219.42 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem diff --git a/devops/tasks/hosts.py b/devops/tasks/hosts.py index e33708360..e7bec0383 100644 --- a/devops/tasks/hosts.py +++ b/devops/tasks/hosts.py @@ -1,18 +1,30 @@ import orchlib.aws +import orchlib.config +import orchlib.fabric_flock +import orchlib.templates +import orchlib.ubuntu +import orchlib.repos +from orchlib.logger import system def update_hosts_ini(): # Fetch list of instances instances = orchlib.aws.list_instances() - + ###print(f"Debug: instances contains: {instances}") + # Open or create the hosts.ini file with open('hosts.ini', 'w') as hosts_file: # Write a default header or group for the ansible inventory hosts_file.write('[aws_servers]\n') - # Iterate through the instances and write their IP addresses + key_file = orchlib.config.creds['key_filename'] + + # Iterate through the instances and write their details for instance in instances: - # Assuming the instance object has an attribute 'ip_address' - hosts_file.write(f"{instance.ip_address}\n") + ## print(f"Debug: instance contains: {instance}") + + instance_name = instance['Tags']['Name'] + ip_address = instance['PublicIpAddress'] + hosts_file.write(f"{instance_name} ansible_host={ip_address} ansible_user=ubuntu ansible_ssh_private_key_file={key_file}\n") # Call the function to update the hosts.ini file update_hosts_ini() \ No newline at end of file From 0c010415636799b2d85ba252256922c8a5a2467c Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 16:46:43 -0400 Subject: [PATCH 15/16] dynamic_hosts.py created --- devops/ansible/README.md | 20 +++++++++++++++++ devops/tasks/dynamic_hosts.py | 41 +++++++++++++++++++++++++++++++++++ devops/tasks/hosts.ini | 2 +- devops/tasks/hosts.py | 2 +- 4 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 devops/tasks/dynamic_hosts.py diff --git a/devops/ansible/README.md b/devops/ansible/README.md index 1ea808a65..b212be774 100644 --- a/devops/ansible/README.md +++ b/devops/ansible/README.md @@ -39,3 +39,23 @@ To setup a new server, do the following: ansible-playbook -i hosts.ini ../ansible/tasks/master_playbook.yaml --limit daclassroom inv certbot daclassroom ansible-playbook -i hosts.ini ../ansible/tasks/reboot.yaml --limit daclassroom + ansible-playbook -i hosts.ini ../ansible/tasks/G_download_config.yaml --limit daclassroom + + The hosts.ini file can be created by running python hosts.py in the tasks directory. + To remove the hosts.ini file completely, you can call the dynamic_hosts.py script. First make sure the script is executable: + chmod +x ./dynamic_hosts.py + + Make sure you're logged into aws by running: + aws sso login + + Then call a playbook usign the dyanmic_hosts.py script like this: + + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/B_install_baseline.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/C_install_repos.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/D_copy_files.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/E_setup_os_environment_variables.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/F_run_additional_tasks.yaml --limit coglabs + inv certbot daclassroom + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/reboot.yaml --limit coglabs + ansible-playbook -i dynamic_hosts.py ../ansible/tasks/G_download_config.yaml --limit coglabs + diff --git a/devops/tasks/dynamic_hosts.py b/devops/tasks/dynamic_hosts.py new file mode 100644 index 000000000..e6e832675 --- /dev/null +++ b/devops/tasks/dynamic_hosts.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json +import orchlib.aws +import orchlib.config + +def get_inventory(): + # Fetch list of instances + instances = orchlib.aws.list_instances() + + # Initialize inventory structure + inventory = { + 'servers': { + 'hosts': [], + 'vars': { + 'ansible_user': 'ubuntu', + 'ansible_ssh_private_key_file': orchlib.config.creds['key_filename'] + } + }, + '_meta': { + 'hostvars': {} + } + } + + # Iterate through the instances and add them to the inventory + for instance in instances: + instance_name = instance['Tags']['Name'] + ip_address = instance['PublicIpAddress'] + + # Add instance to the aws_servers group + inventory['aws_servers']['hosts'].append(instance_name) + + # Add host-specific variables + inventory['_meta']['hostvars'][instance_name] = { + 'ansible_host': ip_address + } + + return inventory + +if __name__ == '__main__': + inventory = get_inventory() + print(json.dumps(inventory, indent=2)) \ No newline at end of file diff --git a/devops/tasks/hosts.ini b/devops/tasks/hosts.ini index a27d50cec..081c06e7c 100644 --- a/devops/tasks/hosts.ini +++ b/devops/tasks/hosts.ini @@ -1,4 +1,4 @@ -[aws_servers] +[servers] open-house ansible_host=3.87.42.157 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem da-i18n ansible_host=54.166.118.68 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem da ansible_host=44.202.219.209 ansible_user=ubuntu ansible_ssh_private_key_file=/root/.ssh/pbrostaws.pem diff --git a/devops/tasks/hosts.py b/devops/tasks/hosts.py index e7bec0383..1637ad2ac 100644 --- a/devops/tasks/hosts.py +++ b/devops/tasks/hosts.py @@ -14,7 +14,7 @@ def update_hosts_ini(): # Open or create the hosts.ini file with open('hosts.ini', 'w') as hosts_file: # Write a default header or group for the ansible inventory - hosts_file.write('[aws_servers]\n') + hosts_file.write('[servers]\n') key_file = orchlib.config.creds['key_filename'] From 5753b71f55fde655d8b593ee87ab60bc2568ac5f Mon Sep 17 00:00:00 2001 From: "Brost, Paul" Date: Mon, 30 Sep 2024 16:49:43 -0400 Subject: [PATCH 16/16] fixed dynamic_hosts.py to work with scripts --- devops/tasks/dynamic_hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/tasks/dynamic_hosts.py b/devops/tasks/dynamic_hosts.py index e6e832675..f1beca03f 100644 --- a/devops/tasks/dynamic_hosts.py +++ b/devops/tasks/dynamic_hosts.py @@ -27,7 +27,7 @@ def get_inventory(): ip_address = instance['PublicIpAddress'] # Add instance to the aws_servers group - inventory['aws_servers']['hosts'].append(instance_name) + inventory['servers']['hosts'].append(instance_name) # Add host-specific variables inventory['_meta']['hostvars'][instance_name] = {