diff --git a/README.md b/README.md index 475b796..a0a7a3d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ ## Task 1 -- [INSTRUCTION.md](./task_1/INSTRUCTION.md) -- [AUTOMATION.md](./task_1/AUTOMATION.md) +- [INSTRUCTION.md](./ansible-hadoop/INSTRUCTION.md) +- [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md) +## Task 2 + +Для второго задания мы доработали нашу ansible-автоматику. Инструкция для неё +всё так же лежит в [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md), там же +описано и получение доступа к эндпоинтам кластера. + +Полностью изменения для второй задачи можно увидеть в [этом +PR](https://github.com/5krotov/BigDataCourse/pull/2) (их там много, но это +из-за того, что папка с ansible-кодом была переназвана и были пофикшены мелкие +баги). diff --git a/task_1/.gitignore b/ansible-hadoop/.gitignore similarity index 100% rename from task_1/.gitignore rename to ansible-hadoop/.gitignore diff --git a/ansible-hadoop/AUTOMATION.md b/ansible-hadoop/AUTOMATION.md new file mode 100644 index 0000000..f33ca9e --- /dev/null +++ b/ansible-hadoop/AUTOMATION.md @@ -0,0 +1,67 @@ +# Автоматизация + +Сделана с помощью ansible. + +![](./src/cluster.svg) + +## Запуск + +Обновите параметры подключения к jump-ноде в инвенторке: + +```bash +cd inventory +vim hosts.yml +``` + +Находясь в той же директории, создайте вольты с паролем от пользователя +`ubuntu`: + +```bash +ansible-vault create ./group_vars/nodes/vault.yml +# ansible_ssh_pass: "PASSWORD" +ansible-vault create ./group_vars/all/vault.yml +# ansible_become_pass: "PASSWORD" +``` + +Теперь можно запускать плейбук, который настроит ВМ и установит на них +hadoop: + +```bash +ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml +``` + +## Доступ к веб-интерфейсам hadoop + +Необходимые сервисы расположены на портах 9870, 8088 и 19888 нейм-ноды, для +доступа вам нужно подключиться к jump-ноде с помощью консольной команды: + +```bash +ssh \ + -L 9870:192.168.10.25:9870 \ + -L 8088:192.168.10.25:8088 \ + -L 19888:192.168.10.25:19888 \ + ubuntu@178.236.25.103 +``` + +Пока туннель активен, эндпоинты кластера будут доступны по ссылам: + +- **HDFS** -- [http://localhost:9870](http://localhost:9870) +- **Resource Manager** -- [http://localhost:8088](http://localhost:8088) +- **JobHistoryServer** -- [http://localhost:19888](http://localhost:19888) + +Однако для удобства можете настроить ssh-конфиг. + +### Настройка ssh-конфига + +```config +Host big-data-jn + HostName 178.236.25.103 + User ubuntu + IdentityFile ~/.ssh/id_rsa + LocalForward 9870 192.168.10.25:9870 + LocalForward 8088 192.168.10.25:8088 + LocalForward 19888 192.168.10.25:19888 +``` + +Далее -- `ssh big-data-jn`. + diff --git a/task_1/INSTRUCTION.md b/ansible-hadoop/INSTRUCTION.md similarity index 100% rename from task_1/INSTRUCTION.md rename to ansible-hadoop/INSTRUCTION.md diff --git a/task_1/ansible/playbooks/hadoop.yml b/ansible-hadoop/ansible/playbooks/hadoop.yml similarity index 72% rename from task_1/ansible/playbooks/hadoop.yml rename to ansible-hadoop/ansible/playbooks/hadoop.yml index 7624a1e..4ae4d8e 100644 --- a/task_1/ansible/playbooks/hadoop.yml +++ b/ansible-hadoop/ansible/playbooks/hadoop.yml @@ -1,6 +1,6 @@ --- -- name: Test play +- name: Set up Hadoop hosts: "all" become: true roles: diff --git a/task_1/ansible/roles/hadoop/defaults/main.yml b/ansible-hadoop/ansible/roles/hadoop/defaults/main.yml similarity index 100% rename from task_1/ansible/roles/hadoop/defaults/main.yml rename to ansible-hadoop/ansible/roles/hadoop/defaults/main.yml diff --git a/task_1/ansible/roles/hadoop/handlers/main.yml b/ansible-hadoop/ansible/roles/hadoop/handlers/main.yml similarity index 100% rename from task_1/ansible/roles/hadoop/handlers/main.yml rename to ansible-hadoop/ansible/roles/hadoop/handlers/main.yml diff --git a/task_1/ansible/roles/hadoop/tasks/configure.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml similarity index 74% rename from task_1/ansible/roles/hadoop/tasks/configure.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml index 6440e89..a3b2eef 100644 --- a/task_1/ansible/roles/hadoop/tasks/configure.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml @@ -3,6 +3,7 @@ - name: Get NameNode IP address set_fact: namenode_ip: "{{ hostvars[groups['name_nodes'][0]].ansible_default_ipv4.address | default(hostvars[groups['name_nodes'][0]].ansible_host) }}" + namenode_hostname: "{{ hostvars[groups['name_nodes'][0]].ansible_hostname }}" run_once: true when: groups['name_nodes'] is defined and groups['name_nodes'] | length > 0 tags: core_site @@ -14,7 +15,7 @@ when: groups['name_nodes'] is not defined or groups['name_nodes'] | length == 0 tags: core_site -- name: Configure SSH client settings for hadoop user +- name: Template hadoop configs template: src: "{{ item }}.j2" dest: "/home/hadoop/hadoop/etc/hadoop/{{ item }}" @@ -23,10 +24,14 @@ loop: - hdfs-site.xml - core-site.xml + - mapred-site.xml + - yarn-site.xml - workers tags: - core_site - hdfs_site + - mapred_site + - yarn_site - workers - name: Display workers file content @@ -108,33 +113,3 @@ msg: "{{ format_result }}" tags: format_namenode -- name: Check if HDFS is already running - command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report - register: hdfs_status - changed_when: false - failed_when: false - when: inventory_hostname == groups['name_nodes'][0] - tags: start_hdfs - -- name: Start HDFS cluster - command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh - args: - chdir: "/home/hadoop/hadoop" - when: - - inventory_hostname == groups['name_nodes'][0] - - hdfs_status.rc != 0 - tags: start_hdfs - -- name: Wait for HDFS services to start - wait_for: - port: "{{ item.port }}" - host: "{{ item.host }}" - delay: 5 - timeout: 60 - state: started - when: inventory_hostname == groups['name_nodes'][0] - loop: - - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode web UI - - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode RPC - tags: start_hdfs - diff --git a/task_1/ansible/roles/hadoop/tasks/install.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/install.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/install.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/install.yml diff --git a/task_1/ansible/roles/hadoop/tasks/main.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml similarity index 81% rename from task_1/ansible/roles/hadoop/tasks/main.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/main.yml index 6615830..f375e74 100644 --- a/task_1/ansible/roles/hadoop/tasks/main.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml @@ -12,6 +12,10 @@ file: configure.yml tags: configure +- import_tasks: + file: start.yml + tags: start + - import_tasks: file: verify.yml tags: verify diff --git a/task_1/ansible/roles/hadoop/tasks/prepare.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/prepare.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/prepare.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/prepare.yml diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml new file mode 100644 index 0000000..7dd5e78 --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml @@ -0,0 +1,95 @@ +--- + +# HDFS # + +- name: Check if HDFS is already running + command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report + register: hdfs_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_hdfs + +- name: Start HDFS cluster + command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - hdfs_status.rc != 0 + tags: start_hdfs + +- name: Wait for HDFS services to start + wait_for: + port: "{{ item.port }}" + host: "{{ item.host }}" + delay: 5 + timeout: 60 + state: started + when: inventory_hostname == groups['name_nodes'][0] + loop: + - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode web UI + - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode RPC + tags: start_hdfs + +# YARN # + +- name: Check if YARN ResourceManager is already running + shell: sudo -u hadoop jps | grep -E 'ResourceManager|NodeManager' || true + register: yarn_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_yarn + +- name: Start YARN cluster + command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-yarn.sh + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - yarn_status.stdout_lines | length == 0 + tags: start_yarn + +- name: Wait for YARN ResourceManager to start + wait_for: + port: "{{ item.port }}" + host: "{{ item.host }}" + delay: 5 + timeout: 60 + state: started + loop: + - { port: 8088, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # RM Web UI + - { port: 8032, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # RM RPC (опционально) + when: inventory_hostname == groups['name_nodes'][0] + tags: start_yarn + +# MapReduce JobHistoryServer # + +- name: Check if JobHistoryServer is already running + shell: sudo -u hadoop jps | grep JobHistoryServer || true + register: history_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_historyserver + +- name: Start JobHistoryServer + command: sudo -u hadoop /home/hadoop/hadoop/bin/mapred --daemon start historyserver + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - history_status.stdout_lines | length == 0 + tags: start_historyserver + +- name: Wait for JobHistoryServer to start + wait_for: + port: 19888 + host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" + delay: 5 + timeout: 60 + state: started + when: inventory_hostname == groups['name_nodes'][0] + tags: start_historyserver + diff --git a/task_1/ansible/roles/hadoop/tasks/verify.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml similarity index 69% rename from task_1/ansible/roles/hadoop/tasks/verify.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml index acac1f3..d3cd583 100644 --- a/task_1/ansible/roles/hadoop/tasks/verify.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml @@ -5,19 +5,19 @@ jps | grep -E 'NameNode|DataNode|SecondaryNameNode' register: jps_output changed_when: false + when: inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes']) tags: verify_hdfs - name: Display HDFS processes debug: msg: "{{ inventory_hostname }}: {{ jps_output.stdout_lines }}" - when: jps_output.stdout_lines | length > 0 + when: + - inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes']) + - jps_output.stdout_lines | length > 0 tags: verify_hdfs - name: Get HDFS report from NameNode - become: yes - become_user: hadoop - shell: | - /home/hadoop/hadoop/bin/hdfs dfsadmin -report + command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report register: hdfs_report changed_when: false when: inventory_hostname == groups['name_nodes'][0] diff --git a/task_1/ansible/roles/hadoop/templates/core-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/core-site.xml.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/core-site.xml.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/core-site.xml.j2 diff --git a/task_1/ansible/roles/hadoop/templates/hdfs-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/hdfs-site.xml.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/hdfs-site.xml.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/hdfs-site.xml.j2 diff --git a/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 new file mode 100644 index 0000000..c74022c --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 @@ -0,0 +1,29 @@ + + + + + + + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + $HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/* + + + diff --git a/task_1/ansible/roles/hadoop/templates/ssh_config.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/ssh_config.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/ssh_config.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/ssh_config.j2 diff --git a/task_1/ansible/roles/hadoop/templates/workers.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/workers.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/workers.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/workers.j2 diff --git a/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 new file mode 100644 index 0000000..699ea2f --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 @@ -0,0 +1,38 @@ + + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME + + + yarn.resourcemanager.hostname + {{ namenode_hostname }} + + + yarn.resourcemanager.address + {{ namenode_hostname }}:8032 + + + yarn.resourcemanager.resource-tracker.address + {{ namenode_hostname }}:8031 + + + diff --git a/task_1/inventory/ansible.cfg b/ansible-hadoop/inventory/ansible.cfg similarity index 100% rename from task_1/inventory/ansible.cfg rename to ansible-hadoop/inventory/ansible.cfg diff --git a/task_1/inventory/group_vars/all/.gitkeep b/ansible-hadoop/inventory/group_vars/all/.gitkeep similarity index 100% rename from task_1/inventory/group_vars/all/.gitkeep rename to ansible-hadoop/inventory/group_vars/all/.gitkeep diff --git a/task_1/inventory/group_vars/nodes/.gitkeep b/ansible-hadoop/inventory/group_vars/nodes/.gitkeep similarity index 100% rename from task_1/inventory/group_vars/nodes/.gitkeep rename to ansible-hadoop/inventory/group_vars/nodes/.gitkeep diff --git a/task_1/inventory/hosts.yml b/ansible-hadoop/inventory/hosts.yml similarity index 100% rename from task_1/inventory/hosts.yml rename to ansible-hadoop/inventory/hosts.yml diff --git a/task_1/run.sh b/ansible-hadoop/run.sh similarity index 100% rename from task_1/run.sh rename to ansible-hadoop/run.sh diff --git a/ansible-hadoop/src/big-data-task-1.drawio b/ansible-hadoop/src/big-data-task-1.drawio new file mode 100644 index 0000000..3063be9 --- /dev/null +++ b/ansible-hadoop/src/big-data-task-1.drawio @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ansible-hadoop/src/cluster.svg b/ansible-hadoop/src/cluster.svg new file mode 100644 index 0000000..cf6c98d --- /dev/null +++ b/ansible-hadoop/src/cluster.svg @@ -0,0 +1,4 @@ + + + +
Edge Node

178.236.25.103
192.168.10.53
Name Node

192.168.10.25
Data Node 0

192.168.10.23
Data Node 1

192.168.10.24
Deploy Node

(can not be = to Edge Node)
 \ No newline at end of file diff --git a/task_1/AUTOMATION.md b/task_1/AUTOMATION.md deleted file mode 100644 index cb06002..0000000 --- a/task_1/AUTOMATION.md +++ /dev/null @@ -1,29 +0,0 @@ -Сделана с помощью ansible. - -![](./src/cluster.png) - -Для запуска автоматизации обновите параметры подключения к jump-ноде в -инвенторке: - -```bash -cd inventory -vim hosts.yml -``` - -А также, находясь в той же директории, создайте вольты с паролем от -пользователя `ubuntu`: - -```bash -ansible-vault create ./group_vars/nodes/vault.yml -# ansible_ssh_pass: "PASSWORD" -ansible-vault create ./group_vars/all/vault.yml -# ansible_become_pass: "PASSWORD" -``` - -Теперь можно запускать плейбук, который настроит ВМ и установит на них -hdfs-кластер: - -```bash -ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml -``` - diff --git a/task_1/src/cluster.png b/task_1/src/cluster.png deleted file mode 100644 index 590d286..0000000 Binary files a/task_1/src/cluster.png and /dev/null differ