From c39d741af3bb533879ef12eae7cb74c3c9e93041 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 24 May 2021 14:59:44 +0530 Subject: [PATCH 1/7] add feature of uploading edx services log to s3 bucket --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 29eebfe7cc2..090806c6f2b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -13,4 +13,9 @@ postrotate /usr/bin/killall -HUP rsyslogd endscript + lastaction + {% if COMMON_OBJECT_STORE_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + {% endif -%} + endscript } From d8f4569f5f99d38c98cb6f91debdf70f4b9c1ea9 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 31 May 2021 11:12:09 +0530 Subject: [PATCH 2/7] add edx services own flag to upload logs --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- playbooks/roles/common_vars/defaults/main.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 090806c6f2b..796329e6fb0 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,8 +14,8 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" {% endif -%} endscript } diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 5d6515e43e5..6cdbe86c5f0 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,6 +23,9 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" +COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_LOG_EDX_SYNC : False + COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" COMMON_APP_DIR: "{{ COMMON_BASE_DIR}}/app" From 2c46feee0df06de6126a4fc74f906b0b980635f1 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:01:16 +0530 Subject: [PATCH 3/7] rename flag to more appropriate name --- playbooks/roles/common_vars/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common_vars/defaults/main.yml b/playbooks/roles/common_vars/defaults/main.yml index 6cdbe86c5f0..f8b64a58dea 100644 --- a/playbooks/roles/common_vars/defaults/main.yml +++ b/playbooks/roles/common_vars/defaults/main.yml @@ -23,8 +23,8 @@ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/" COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store" COMMON_OBJECT_STORE_LOG_SYNC_ON_EXIT: "{{ COMMON_BIN_DIR }}/sync-logs-on-exit" -COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX : "logs/edx/" -COMMON_OBJECT_STORE_LOG_EDX_SYNC : False +COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX : "logs/edx/" +COMMON_OBJECT_STORE_EDX_LOG_SYNC : False COMMON_BASE_DIR: /edx COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var" From 88ca71b67a4e97706548a3dfae95c695baa292fc Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:02:47 +0530 Subject: [PATCH 4/7] fix the edx serverices log folder --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 796329e6fb0..92de9dbd261 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -14,8 +14,9 @@ /usr/bin/killall -HUP rsyslogd endscript lastaction - {% if COMMON_OBJECT_STORE_LOG_EDX_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/edx" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_EDX_PREFIX }}" + {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" {% endif -%} endscript } From e35e8230e6bd96ddd231ff0c484e14cdd43f6abd Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:29:27 +0530 Subject: [PATCH 5/7] handle the case where file name same in both the services --- .../common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 index 92de9dbd261..c8528b25c7b 100644 --- a/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 +++ b/playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate.j2 @@ -15,8 +15,8 @@ endscript lastaction {% if COMMON_OBJECT_STORE_EDX_LOG_SYNC -%} - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" - {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/lms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/cms" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" {% endif -%} endscript } From 7b2c1c84e4260518ca04a6ed0e60a376c24609b6 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Thu, 3 Jun 2021 20:30:56 +0530 Subject: [PATCH 6/7] add commands to sync the edx services logs --- playbooks/retire_host.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 32fddd184f1..9684c8ec9ae 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,8 +37,10 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking" - + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" + command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + - name: Run minos verification hosts: "{{TARGET}}" become: True From 1d6bf9060823c520cdea32031f2f2c32765a0861 Mon Sep 17 00:00:00 2001 From: Sandeep Kumar Choudhary Date: Mon, 7 Jun 2021 17:00:29 +0530 Subject: [PATCH 7/7] compress the command with items --- playbooks/retire_host.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/playbooks/retire_host.yml b/playbooks/retire_host.yml index 9684c8ec9ae..19388cc0d59 100644 --- a/playbooks/retire_host.yml +++ b/playbooks/retire_host.yml @@ -37,9 +37,11 @@ - "edx-services" # This catches the case where tracking.log is 0b - name: Sync again - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/lms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/" - command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/cms/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/" + command: "/edx/bin/send-logs-to-object-store -d {{ COMMON_LOG_DIR }}{{ item.src }} -b {{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }} -p {{ item.dest }}" + with_items: + - { src: '/tracking/', dest: '{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}' } + - { src: '/lms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}lms/' } + - { src: '/cms/', dest: '{{ COMMON_OBJECT_STORE_EDX_LOG_SYNC_PREFIX }}cms/' } - name: Run minos verification hosts: "{{TARGET}}"