Hw 2

README.md

@@ -2,6 +2,16 @@
 
 ## Task 1
 
-- [INSTRUCTION.md](./task_1/INSTRUCTION.md)
-- [AUTOMATION.md](./task_1/AUTOMATION.md)
+- [INSTRUCTION.md](./ansible-hadoop/INSTRUCTION.md)
+- [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md)
 
+## Task 2
+
+Для второго задания мы доработали нашу ansible-автоматику. Инструкция для неё
+всё так же лежит в [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md), там же
+описано и получение доступа к эндпоинтам кластера.
+
+Полностью изменения для второй задачи можно увидеть в [этом
+PR](https://github.com/5krotov/BigDataCourse/pull/2) (их там много, но это
+из-за того, что папка с ansible-кодом была переназвана и были пофикшены мелкие
+баги).

ansible-hadoop/AUTOMATION.md

@@ -0,0 +1,67 @@
+# Автоматизация
+
+Сделана с помощью ansible.
+
+![](./src/cluster.svg)
+
+## Запуск
+
+Обновите параметры подключения к jump-ноде в инвенторке:
+
+```bash
+cd inventory
+vim hosts.yml
+```
+
+Находясь в той же директории, создайте вольты с паролем от пользователя
+`ubuntu`:
+
+```bash
+ansible-vault create ./group_vars/nodes/vault.yml
+# ansible_ssh_pass: "PASSWORD"
+ansible-vault create ./group_vars/all/vault.yml
+# ansible_become_pass: "PASSWORD"
+```
+
+Теперь можно запускать плейбук, который настроит ВМ и установит на них
+hadoop:
+
+```bash
+ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml
+```
+
+## Доступ к веб-интерфейсам hadoop
+
+Необходимые сервисы расположены на портах 9870, 8088 и 19888 нейм-ноды, для
+доступа вам нужно подключиться к jump-ноде с помощью консольной команды:
+
+```bash
+ssh \
+    -L 9870:192.168.10.25:9870 \
+    -L 8088:192.168.10.25:8088 \
+    -L 19888:192.168.10.25:19888 \
+    ubuntu@178.236.25.103
+```
+
+Пока туннель активен, эндпоинты кластера будут доступны по ссылам:
+
+- **HDFS** -- [http://localhost:9870](http://localhost:9870)
+- **Resource Manager** -- [http://localhost:8088](http://localhost:8088)
+- **JobHistoryServer** -- [http://localhost:19888](http://localhost:19888)
+
+Однако для удобства можете настроить ssh-конфиг.
+
+### Настройка ssh-конфига
+
+```config
+Host big-data-jn
+  HostName 178.236.25.103 
+  User ubuntu
+  IdentityFile ~/.ssh/id_rsa
+  LocalForward 9870 192.168.10.25:9870
+  LocalForward 8088 192.168.10.25:8088
+  LocalForward 19888 192.168.10.25:19888
+```
+
+Далее -- `ssh big-data-jn`.
+

task_1/ansible/playbooks/hadoop.yml → ansible-hadoop/ansible/playbooks/hadoop.yml

@@ -1,6 +1,6 @@
 ---
 
-- name: Test play
+- name: Set up Hadoop
   hosts: "all"
   become: true
   roles:

task_1/ansible/roles/hadoop/tasks/configure.yml → ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml

@@ -3,6 +3,7 @@
 - name: Get NameNode IP address
   set_fact:
     namenode_ip: "{{ hostvars[groups['name_nodes'][0]].ansible_default_ipv4.address | default(hostvars[groups['name_nodes'][0]].ansible_host) }}"
+    namenode_hostname: "{{ hostvars[groups['name_nodes'][0]].ansible_hostname }}"
   run_once: true
   when: groups['name_nodes'] is defined and groups['name_nodes'] | length > 0
   tags: core_site
@@ -14,7 +15,7 @@
   when: groups['name_nodes'] is not defined or groups['name_nodes'] | length == 0
   tags: core_site
 
-- name: Configure SSH client settings for hadoop user
+- name: Template hadoop configs
   template:
     src: "{{ item }}.j2"
     dest: "/home/hadoop/hadoop/etc/hadoop/{{ item }}"
@@ -23,10 +24,14 @@
   loop:
     - hdfs-site.xml
     - core-site.xml
+    - mapred-site.xml
+    - yarn-site.xml
     - workers
   tags:
     - core_site
     - hdfs_site
+    - mapred_site
+    - yarn_site
     - workers
 
 - name: Display workers file content
@@ -108,33 +113,3 @@
     msg: "{{ format_result }}"
   tags: format_namenode
 
-- name: Check if HDFS is already running
-  command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report
-  register: hdfs_status
-  changed_when: false
-  failed_when: false
-  when: inventory_hostname == groups['name_nodes'][0]
-  tags: start_hdfs
-
-- name: Start HDFS cluster
-  command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh
-  args:
-    chdir: "/home/hadoop/hadoop"
-  when: 
-    - inventory_hostname == groups['name_nodes'][0]
-    - hdfs_status.rc != 0
-  tags: start_hdfs
-
-- name: Wait for HDFS services to start
-  wait_for:
-    port: "{{ item.port }}"
-    host: "{{ item.host }}"
-    delay: 5
-    timeout: 60
-    state: started
-  when: inventory_hostname == groups['name_nodes'][0]
-  loop:
-    - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # NameNode web UI
-    - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # NameNode RPC
-  tags: start_hdfs
-

task_1/ansible/roles/hadoop/tasks/main.yml → ansible-hadoop/ansible/roles/hadoop/tasks/main.yml

@@ -12,6 +12,10 @@
     file: configure.yml
   tags: configure
 
+- import_tasks:
+    file: start.yml
+  tags: start
+
 - import_tasks:
     file: verify.yml
   tags: verify

ansible-hadoop/ansible/roles/hadoop/tasks/start.yml

@@ -0,0 +1,95 @@
+---
+
+# HDFS #
+
+- name: Check if HDFS is already running
+  command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report
+  register: hdfs_status
+  changed_when: false
+  failed_when: false
+  when: inventory_hostname == groups['name_nodes'][0]
+  tags: start_hdfs
+
+- name: Start HDFS cluster
+  command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh
+  args:
+    chdir: "/home/hadoop/hadoop"
+  when:
+    - inventory_hostname == groups['name_nodes'][0]
+    - hdfs_status.rc != 0
+  tags: start_hdfs
+
+- name: Wait for HDFS services to start
+  wait_for:
+    port: "{{ item.port }}"
+    host: "{{ item.host }}"
+    delay: 5
+    timeout: 60
+    state: started
+  when: inventory_hostname == groups['name_nodes'][0]
+  loop:
+    - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # NameNode web UI
+    - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # NameNode RPC
+  tags: start_hdfs
+
+# YARN #
+
+- name: Check if YARN ResourceManager is already running
+  shell: sudo -u hadoop jps | grep -E 'ResourceManager|NodeManager' || true
+  register: yarn_status
+  changed_when: false
+  failed_when: false
+  when: inventory_hostname == groups['name_nodes'][0]
+  tags: start_yarn
+
+- name: Start YARN cluster
+  command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-yarn.sh
+  args:
+    chdir: "/home/hadoop/hadoop"
+  when:
+    - inventory_hostname == groups['name_nodes'][0]
+    - yarn_status.stdout_lines | length == 0
+  tags: start_yarn
+
+- name: Wait for YARN ResourceManager to start
+  wait_for:
+    port: "{{ item.port }}"
+    host: "{{ item.host }}"
+    delay: 5
+    timeout: 60
+    state: started
+  loop:
+    - { port: 8088, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # RM Web UI
+    - { port: 8032, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" }  # RM RPC (опционально)
+  when: inventory_hostname == groups['name_nodes'][0]
+  tags: start_yarn
+
+# MapReduce JobHistoryServer #
+
+- name: Check if JobHistoryServer is already running
+  shell: sudo -u hadoop jps | grep JobHistoryServer || true
+  register: history_status
+  changed_when: false
+  failed_when: false
+  when: inventory_hostname == groups['name_nodes'][0]
+  tags: start_historyserver
+
+- name: Start JobHistoryServer
+  command: sudo -u hadoop /home/hadoop/hadoop/bin/mapred --daemon start historyserver
+  args:
+    chdir: "/home/hadoop/hadoop"
+  when:
+    - inventory_hostname == groups['name_nodes'][0]
+    - history_status.stdout_lines | length == 0
+  tags: start_historyserver
+
+- name: Wait for JobHistoryServer to start
+  wait_for:
+    port: 19888
+    host: "{{ ansible_default_ipv4.address | default(ansible_host) }}"
+    delay: 5
+    timeout: 60
+    state: started
+  when: inventory_hostname == groups['name_nodes'][0]
+  tags: start_historyserver
+

task_1/ansible/roles/hadoop/tasks/verify.yml → ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml

@@ -5,19 +5,19 @@
     jps | grep -E 'NameNode|DataNode|SecondaryNameNode'
   register: jps_output
   changed_when: false
+  when: inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes'])
   tags: verify_hdfs
 
 - name: Display HDFS processes
   debug:
     msg: "{{ inventory_hostname }}: {{ jps_output.stdout_lines }}"
-  when: jps_output.stdout_lines | length > 0
+  when:
+    - inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes'])
+    - jps_output.stdout_lines | length > 0
   tags: verify_hdfs
 
 - name: Get HDFS report from NameNode
-  become: yes
-  become_user: hadoop
-  shell: |
-    /home/hadoop/hadoop/bin/hdfs dfsadmin -report
+  command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report
   register: hdfs_report
   changed_when: false
   when: inventory_hostname == groups['name_nodes'][0]

ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2

@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+  <property>
+    <name>mapreduce.framework.name</name>
+    <value>yarn</value>
+  </property>
+  <property>
+    <name>mapreduce.application.classpath</name>
+    <value>$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*</value>
+  </property>
+</configuration>
+

ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2

@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.env-whitelist</name>
+        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
+    </property>
+    <property>
+        <name>yarn.resourcemanager.hostname</name>
+        <value>{{ namenode_hostname }}</value>
+    </property>
+    <property>
+        <name>yarn.resourcemanager.address</name>
+        <value>{{ namenode_hostname }}:8032</value>
+    </property>
+    <property>
+        <name>yarn.resourcemanager.resource-tracker.address</name>
+        <value>{{ namenode_hostname }}:8031</value>
+    </property>
+</configuration>
+