From 233d3b02f36807f6b82944a5b9e5875ed0ae14be Mon Sep 17 00:00:00 2001 From: Nick Poyant Date: Sat, 17 Aug 2019 23:05:42 -0400 Subject: [PATCH 1/3] Add directory containing nvme disk fix automation --- openshift-aws-nvme-disk-order-fix/README.md | 17 ++++++++++ .../ocp-aws-nvme-docker-postinstall-fix.yml | 33 +++++++++++++++++++ .../ocp-aws-nvme-docker-preinstall.yml | 32 ++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 openshift-aws-nvme-disk-order-fix/README.md create mode 100644 openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml create mode 100644 openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml diff --git a/openshift-aws-nvme-disk-order-fix/README.md b/openshift-aws-nvme-disk-order-fix/README.md new file mode 100644 index 0000000..669e5c0 --- /dev/null +++ b/openshift-aws-nvme-disk-order-fix/README.md @@ -0,0 +1,17 @@ +# README + +Description: + +It was discovered in an OpenShift Cluster deployed into AWS that the AWS NVME disk order changes after reboot in the Red Hat Enterprise Linux 7 operating system. This impacts docker's storage configuration negatively if you have multiple disks configured. This repo contains two basic playbooks that address the two core use cases. + +# Use Case 1: Pre Docker Storage Setup + +If you are configuring docker storage for the first time: + +ocp-aws-nvme-docker-preinstall.yml + +# Use Case 2: Post Docker storage setup + +If you have already configured docker storage, disks have changed order, and you need to fix dockers configuration in order to bring up the service: + +ocp-aws-nvme-docker-preinstall-fix.yml diff --git a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml new file mode 100644 index 0000000..337c15f --- /dev/null +++ b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml @@ -0,0 +1,33 @@ +--- +- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS + hosts: all + tasks: + - name: Determine disk used for PV in docker-vg volume group + shell: vgdisplay docker-vg -v | grep "PV Name" -A1 | awk '{print $3}' | sed -n '1p' | sed 's/.\{2\}$//' + register: UUID + + - debug: msg={{ UUID.stdout }} + + - name: Grab NVME Path from udevinfo info {{ UUID.stdout }} + shell: udevadm info {{ UUID.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' + register: NVME_PATH + + - debug: msg={{ NVME_PATH.stdout }} + + - name: Backup /etc/sysconfig/docker-storage-setup to /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} + copy: + src: /etc/sysconfig/docker-storage-setup + dest: /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} + remote_src: yes + + - name: Update /etc/sysconfig/docker-storage-setup with Persistent Device Patch + lineinfile: + path: /etc/sysconfig/docker-storage-setup + regexp: '^DEVS' + line: 'DEVS=/dev/disk/by-id/{{ NVME_PATH.stdout }}' + + - name: Remove Incorrrect Disk from Volume Group + shell: vgreduce --removemissing docker-vg + + - name: Activate Volume Group + shell: lvchange -a y docker-vg diff --git a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml new file mode 100644 index 0000000..f888092 --- /dev/null +++ b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml @@ -0,0 +1,32 @@ +--- +- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS + hosts: all + vars: + ocp_docker_storage_size: 151 # Define the size of the disk that you want to use for docker storage + tasks: + + - name: Get disk for Docker Storage + shell: "lsblk | grep disk | grep {{ ocp_docker_storage_size }} |cut -f1 -d' '" + register: docker_device + + - name: Grab NVME Path from udevinfo info {{ docker_device.stdout }} + shell: udevadm info /dev/{{ docker_device.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' + register: NVME_PATH + + - name: Configure Docker Storage + copy: + dest: /etc/sysconfig/docker-storage-setup + content: | + DEVS="/dev/disk/by-id/{{ NVME_PATH.stdout }}" + VG=docker-vg + + - name: Docker storage setup + command: docker-storage-setup + + - name: Start and Enable docker + service: + name: docker + state: started + enabled: yes + + From 75812859243d8bd2da18562943cced4d8a104cc2 Mon Sep 17 00:00:00 2001 From: Nick Poyant Date: Sat, 17 Aug 2019 23:08:09 -0400 Subject: [PATCH 2/3] renamed dir --- openshift_aws_nvme_disk_order_fix/README.md | 17 ++++++++++ .../ocp-aws-nvme-docker-postinstall-fix.yml | 33 +++++++++++++++++++ .../ocp-aws-nvme-docker-preinstall.yml | 32 ++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 openshift_aws_nvme_disk_order_fix/README.md create mode 100644 openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-postinstall-fix.yml create mode 100644 openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-preinstall.yml diff --git a/openshift_aws_nvme_disk_order_fix/README.md b/openshift_aws_nvme_disk_order_fix/README.md new file mode 100644 index 0000000..669e5c0 --- /dev/null +++ b/openshift_aws_nvme_disk_order_fix/README.md @@ -0,0 +1,17 @@ +# README + +Description: + +It was discovered in an OpenShift Cluster deployed into AWS that the AWS NVME disk order changes after reboot in the Red Hat Enterprise Linux 7 operating system. This impacts docker's storage configuration negatively if you have multiple disks configured. This repo contains two basic playbooks that address the two core use cases. + +# Use Case 1: Pre Docker Storage Setup + +If you are configuring docker storage for the first time: + +ocp-aws-nvme-docker-preinstall.yml + +# Use Case 2: Post Docker storage setup + +If you have already configured docker storage, disks have changed order, and you need to fix dockers configuration in order to bring up the service: + +ocp-aws-nvme-docker-preinstall-fix.yml diff --git a/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-postinstall-fix.yml b/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-postinstall-fix.yml new file mode 100644 index 0000000..337c15f --- /dev/null +++ b/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-postinstall-fix.yml @@ -0,0 +1,33 @@ +--- +- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS + hosts: all + tasks: + - name: Determine disk used for PV in docker-vg volume group + shell: vgdisplay docker-vg -v | grep "PV Name" -A1 | awk '{print $3}' | sed -n '1p' | sed 's/.\{2\}$//' + register: UUID + + - debug: msg={{ UUID.stdout }} + + - name: Grab NVME Path from udevinfo info {{ UUID.stdout }} + shell: udevadm info {{ UUID.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' + register: NVME_PATH + + - debug: msg={{ NVME_PATH.stdout }} + + - name: Backup /etc/sysconfig/docker-storage-setup to /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} + copy: + src: /etc/sysconfig/docker-storage-setup + dest: /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} + remote_src: yes + + - name: Update /etc/sysconfig/docker-storage-setup with Persistent Device Patch + lineinfile: + path: /etc/sysconfig/docker-storage-setup + regexp: '^DEVS' + line: 'DEVS=/dev/disk/by-id/{{ NVME_PATH.stdout }}' + + - name: Remove Incorrrect Disk from Volume Group + shell: vgreduce --removemissing docker-vg + + - name: Activate Volume Group + shell: lvchange -a y docker-vg diff --git a/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-preinstall.yml b/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-preinstall.yml new file mode 100644 index 0000000..f888092 --- /dev/null +++ b/openshift_aws_nvme_disk_order_fix/ocp-aws-nvme-docker-preinstall.yml @@ -0,0 +1,32 @@ +--- +- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS + hosts: all + vars: + ocp_docker_storage_size: 151 # Define the size of the disk that you want to use for docker storage + tasks: + + - name: Get disk for Docker Storage + shell: "lsblk | grep disk | grep {{ ocp_docker_storage_size }} |cut -f1 -d' '" + register: docker_device + + - name: Grab NVME Path from udevinfo info {{ docker_device.stdout }} + shell: udevadm info /dev/{{ docker_device.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' + register: NVME_PATH + + - name: Configure Docker Storage + copy: + dest: /etc/sysconfig/docker-storage-setup + content: | + DEVS="/dev/disk/by-id/{{ NVME_PATH.stdout }}" + VG=docker-vg + + - name: Docker storage setup + command: docker-storage-setup + + - name: Start and Enable docker + service: + name: docker + state: started + enabled: yes + + From 68e69a7560338c8a82e5a063afa749baf4e83054 Mon Sep 17 00:00:00 2001 From: Nick Poyant Date: Sat, 17 Aug 2019 23:10:39 -0400 Subject: [PATCH 3/3] remove old dir --- openshift-aws-nvme-disk-order-fix/README.md | 17 ---------- .../ocp-aws-nvme-docker-postinstall-fix.yml | 33 ------------------- .../ocp-aws-nvme-docker-preinstall.yml | 32 ------------------ 3 files changed, 82 deletions(-) delete mode 100644 openshift-aws-nvme-disk-order-fix/README.md delete mode 100644 openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml delete mode 100644 openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml diff --git a/openshift-aws-nvme-disk-order-fix/README.md b/openshift-aws-nvme-disk-order-fix/README.md deleted file mode 100644 index 669e5c0..0000000 --- a/openshift-aws-nvme-disk-order-fix/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# README - -Description: - -It was discovered in an OpenShift Cluster deployed into AWS that the AWS NVME disk order changes after reboot in the Red Hat Enterprise Linux 7 operating system. This impacts docker's storage configuration negatively if you have multiple disks configured. This repo contains two basic playbooks that address the two core use cases. - -# Use Case 1: Pre Docker Storage Setup - -If you are configuring docker storage for the first time: - -ocp-aws-nvme-docker-preinstall.yml - -# Use Case 2: Post Docker storage setup - -If you have already configured docker storage, disks have changed order, and you need to fix dockers configuration in order to bring up the service: - -ocp-aws-nvme-docker-preinstall-fix.yml diff --git a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml deleted file mode 100644 index 337c15f..0000000 --- a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-postinstall-fix.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS - hosts: all - tasks: - - name: Determine disk used for PV in docker-vg volume group - shell: vgdisplay docker-vg -v | grep "PV Name" -A1 | awk '{print $3}' | sed -n '1p' | sed 's/.\{2\}$//' - register: UUID - - - debug: msg={{ UUID.stdout }} - - - name: Grab NVME Path from udevinfo info {{ UUID.stdout }} - shell: udevadm info {{ UUID.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' - register: NVME_PATH - - - debug: msg={{ NVME_PATH.stdout }} - - - name: Backup /etc/sysconfig/docker-storage-setup to /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} - copy: - src: /etc/sysconfig/docker-storage-setup - dest: /etc/sysconfig/docker-storage-setup.{{ ansible_date_time.date }}.{{ ansible_date_time.time }} - remote_src: yes - - - name: Update /etc/sysconfig/docker-storage-setup with Persistent Device Patch - lineinfile: - path: /etc/sysconfig/docker-storage-setup - regexp: '^DEVS' - line: 'DEVS=/dev/disk/by-id/{{ NVME_PATH.stdout }}' - - - name: Remove Incorrrect Disk from Volume Group - shell: vgreduce --removemissing docker-vg - - - name: Activate Volume Group - shell: lvchange -a y docker-vg diff --git a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml b/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml deleted file mode 100644 index f888092..0000000 --- a/openshift-aws-nvme-disk-order-fix/ocp-aws-nvme-docker-preinstall.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Update /etc/sysconfig/docker-storage-setup configuration to use a persistent path from AWS - hosts: all - vars: - ocp_docker_storage_size: 151 # Define the size of the disk that you want to use for docker storage - tasks: - - - name: Get disk for Docker Storage - shell: "lsblk | grep disk | grep {{ ocp_docker_storage_size }} |cut -f1 -d' '" - register: docker_device - - - name: Grab NVME Path from udevinfo info {{ docker_device.stdout }} - shell: udevadm info /dev/{{ docker_device.stdout }} | sed -n '3p' | sed 's/^.\{,14\}//' - register: NVME_PATH - - - name: Configure Docker Storage - copy: - dest: /etc/sysconfig/docker-storage-setup - content: | - DEVS="/dev/disk/by-id/{{ NVME_PATH.stdout }}" - VG=docker-vg - - - name: Docker storage setup - command: docker-storage-setup - - - name: Start and Enable docker - service: - name: docker - state: started - enabled: yes - -