From f43906591f1b0ca3fb758b835bf8a41b35c82f64 Mon Sep 17 00:00:00 2001 From: Valery Bergman Date: Thu, 5 Mar 2026 01:44:36 +0300 Subject: [PATCH 1/4] Initial commit --- {task_1 => ansible-hadoop}/.gitignore | 0 {task_1 => ansible-hadoop}/AUTOMATION.md | 0 {task_1 => ansible-hadoop}/INSTRUCTION.md | 0 .../ansible/playbooks/hadoop.yml | 0 .../ansible/roles/hadoop/defaults/main.yml | 0 .../ansible/roles/hadoop/handlers/main.yml | 0 .../ansible/roles/hadoop/tasks/configure.yml | 0 .../ansible/roles/hadoop/tasks/install.yml | 0 .../ansible/roles/hadoop/tasks/main.yml | 0 .../ansible/roles/hadoop/tasks/prepare.yml | 0 .../ansible/roles/hadoop/tasks/verify.yml | 0 .../ansible/roles/hadoop/templates/core-site.xml.j2 | 0 .../ansible/roles/hadoop/templates/hdfs-site.xml.j2 | 0 .../ansible/roles/hadoop/templates/ssh_config.j2 | 0 .../ansible/roles/hadoop/templates/workers.j2 | 0 {task_1 => ansible-hadoop}/inventory/ansible.cfg | 0 .../inventory/group_vars/all/.gitkeep | 0 .../inventory/group_vars/nodes/.gitkeep | 0 {task_1 => ansible-hadoop}/inventory/hosts.yml | 0 {task_1 => ansible-hadoop}/run.sh | 0 {task_1 => ansible-hadoop}/src/cluster.png | Bin 21 files changed, 0 insertions(+), 0 deletions(-) rename {task_1 => ansible-hadoop}/.gitignore (100%) rename {task_1 => ansible-hadoop}/AUTOMATION.md (100%) rename {task_1 => ansible-hadoop}/INSTRUCTION.md (100%) rename {task_1 => ansible-hadoop}/ansible/playbooks/hadoop.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/defaults/main.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/handlers/main.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/tasks/configure.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/tasks/install.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/tasks/main.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/tasks/prepare.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/tasks/verify.yml (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/templates/core-site.xml.j2 (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/templates/hdfs-site.xml.j2 (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/templates/ssh_config.j2 (100%) rename {task_1 => ansible-hadoop}/ansible/roles/hadoop/templates/workers.j2 (100%) rename {task_1 => ansible-hadoop}/inventory/ansible.cfg (100%) rename {task_1 => ansible-hadoop}/inventory/group_vars/all/.gitkeep (100%) rename {task_1 => ansible-hadoop}/inventory/group_vars/nodes/.gitkeep (100%) rename {task_1 => ansible-hadoop}/inventory/hosts.yml (100%) rename {task_1 => ansible-hadoop}/run.sh (100%) rename {task_1 => ansible-hadoop}/src/cluster.png (100%) diff --git a/task_1/.gitignore b/ansible-hadoop/.gitignore similarity index 100% rename from task_1/.gitignore rename to ansible-hadoop/.gitignore diff --git a/task_1/AUTOMATION.md b/ansible-hadoop/AUTOMATION.md similarity index 100% rename from task_1/AUTOMATION.md rename to ansible-hadoop/AUTOMATION.md diff --git a/task_1/INSTRUCTION.md b/ansible-hadoop/INSTRUCTION.md similarity index 100% rename from task_1/INSTRUCTION.md rename to ansible-hadoop/INSTRUCTION.md diff --git a/task_1/ansible/playbooks/hadoop.yml b/ansible-hadoop/ansible/playbooks/hadoop.yml similarity index 100% rename from task_1/ansible/playbooks/hadoop.yml rename to ansible-hadoop/ansible/playbooks/hadoop.yml diff --git a/task_1/ansible/roles/hadoop/defaults/main.yml b/ansible-hadoop/ansible/roles/hadoop/defaults/main.yml similarity index 100% rename from task_1/ansible/roles/hadoop/defaults/main.yml rename to ansible-hadoop/ansible/roles/hadoop/defaults/main.yml diff --git a/task_1/ansible/roles/hadoop/handlers/main.yml b/ansible-hadoop/ansible/roles/hadoop/handlers/main.yml similarity index 100% rename from task_1/ansible/roles/hadoop/handlers/main.yml rename to ansible-hadoop/ansible/roles/hadoop/handlers/main.yml diff --git a/task_1/ansible/roles/hadoop/tasks/configure.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/configure.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml diff --git a/task_1/ansible/roles/hadoop/tasks/install.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/install.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/install.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/install.yml diff --git a/task_1/ansible/roles/hadoop/tasks/main.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/main.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/main.yml diff --git a/task_1/ansible/roles/hadoop/tasks/prepare.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/prepare.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/prepare.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/prepare.yml diff --git a/task_1/ansible/roles/hadoop/tasks/verify.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml similarity index 100% rename from task_1/ansible/roles/hadoop/tasks/verify.yml rename to ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml diff --git a/task_1/ansible/roles/hadoop/templates/core-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/core-site.xml.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/core-site.xml.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/core-site.xml.j2 diff --git a/task_1/ansible/roles/hadoop/templates/hdfs-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/hdfs-site.xml.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/hdfs-site.xml.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/hdfs-site.xml.j2 diff --git a/task_1/ansible/roles/hadoop/templates/ssh_config.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/ssh_config.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/ssh_config.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/ssh_config.j2 diff --git a/task_1/ansible/roles/hadoop/templates/workers.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/workers.j2 similarity index 100% rename from task_1/ansible/roles/hadoop/templates/workers.j2 rename to ansible-hadoop/ansible/roles/hadoop/templates/workers.j2 diff --git a/task_1/inventory/ansible.cfg b/ansible-hadoop/inventory/ansible.cfg similarity index 100% rename from task_1/inventory/ansible.cfg rename to ansible-hadoop/inventory/ansible.cfg diff --git a/task_1/inventory/group_vars/all/.gitkeep b/ansible-hadoop/inventory/group_vars/all/.gitkeep similarity index 100% rename from task_1/inventory/group_vars/all/.gitkeep rename to ansible-hadoop/inventory/group_vars/all/.gitkeep diff --git a/task_1/inventory/group_vars/nodes/.gitkeep b/ansible-hadoop/inventory/group_vars/nodes/.gitkeep similarity index 100% rename from task_1/inventory/group_vars/nodes/.gitkeep rename to ansible-hadoop/inventory/group_vars/nodes/.gitkeep diff --git a/task_1/inventory/hosts.yml b/ansible-hadoop/inventory/hosts.yml similarity index 100% rename from task_1/inventory/hosts.yml rename to ansible-hadoop/inventory/hosts.yml diff --git a/task_1/run.sh b/ansible-hadoop/run.sh similarity index 100% rename from task_1/run.sh rename to ansible-hadoop/run.sh diff --git a/task_1/src/cluster.png b/ansible-hadoop/src/cluster.png similarity index 100% rename from task_1/src/cluster.png rename to ansible-hadoop/src/cluster.png From f911b59af77e9c3b948907a98b95973ac2fbd5f2 Mon Sep 17 00:00:00 2001 From: Valery Bergman Date: Thu, 5 Mar 2026 02:21:03 +0300 Subject: [PATCH 2/4] Update information files --- README.md | 4 +- ansible-hadoop/AUTOMATION.md | 70 ++++++++++++++++++++-- ansible-hadoop/src/big-data-task-1.drawio | 54 +++++++++++++++++ ansible-hadoop/src/cluster.png | Bin 25989 -> 0 bytes ansible-hadoop/src/cluster.svg | 4 ++ 5 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 ansible-hadoop/src/big-data-task-1.drawio delete mode 100644 ansible-hadoop/src/cluster.png create mode 100644 ansible-hadoop/src/cluster.svg diff --git a/README.md b/README.md index 475b796..2b73480 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,6 @@ ## Task 1 -- [INSTRUCTION.md](./task_1/INSTRUCTION.md) -- [AUTOMATION.md](./task_1/AUTOMATION.md) +- [INSTRUCTION.md](./ansible-hadoop/INSTRUCTION.md) +- [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md) diff --git a/ansible-hadoop/AUTOMATION.md b/ansible-hadoop/AUTOMATION.md index cb06002..6e08556 100644 --- a/ansible-hadoop/AUTOMATION.md +++ b/ansible-hadoop/AUTOMATION.md @@ -1,16 +1,19 @@ +# Автоматизация + Сделана с помощью ansible. -![](./src/cluster.png) +![](./src/cluster.svg) + +## Запуск -Для запуска автоматизации обновите параметры подключения к jump-ноде в -инвенторке: +Обновите параметры подключения к jump-ноде в инвенторке: ```bash cd inventory vim hosts.yml ``` -А также, находясь в той же директории, создайте вольты с паролем от +Также, находясь в той же директории, создайте вольты с паролем от пользователя `ubuntu`: ```bash @@ -27,3 +30,62 @@ hdfs-кластер: ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml ``` +## Доступ к веб-интерфейсам hadoop + +Необходимые сервисы расположены на портах 9870, 8088 и 19888 нейм-ноды. Вы можете получить к ним доступ, подключившись к jump-ноде с помощью консольной команды: + +```bash +ssh \ + -L 9870:192.168.10.25:9870 \ + -L 8088:192.168.10.25:8088 \ + -L 19888:192.168.10.25:19888 \ + ubuntu@178.236.25.103 +``` + +Однако для удобства можете настроить ssh-конфиг. + +### Настройка ssh-конфига + +Автоматика использует ваш ключ только для подключения к jump-ноде, далее она +создаёт cluster-ключ, который разносит на все ВМ на стенде. Пользователю этот +ключ не передаётся, поэтому, для получения доступа к веб-интерфейсам у вас есть +два пути, которые будут описаны далее. + +**Первый вариант** -- настроить ssh-config для подключения на нейм-ноду через +джамп: + +```config +Host big-data-jn + HostName 178.236.25.103 + User ubuntu + IdentityFile ~/.ssh/id_rsa + +Host big-data-nn + HostName 192.168.10.25 + User ubuntu + IdentityFile ~/.ssh/id_rsa + LocalForward 9870 localhost:9870 + LocalForward 8088 localhost:8088 + LocalForward 19888 localhost:19888 + ProxyJump big-data-jn +``` + +Для этого будет необходимо внести в authorized_keys нейм-ноды ваш личный ключ. + +Далее -- `ssh big-data-nn`. + +**Второй вариант** -- как на вебинаре, подключаться к jump-ноде и прокидывать +себе порты с интерфейса нейм-ноды: + +```config +Host big-data-jn + HostName 178.236.25.103 + User ubuntu + IdentityFile ~/.ssh/id_rsa + LocalForward 9870 192.168.10.25:9870 + LocalForward 8088 192.168.10.25:8088 + LocalForward 19888 192.168.10.25:19888 +``` + +Далее -- `ssh big-data-jn`. + diff --git a/ansible-hadoop/src/big-data-task-1.drawio b/ansible-hadoop/src/big-data-task-1.drawio new file mode 100644 index 0000000..3063be9 --- /dev/null +++ b/ansible-hadoop/src/big-data-task-1.drawio @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ansible-hadoop/src/cluster.png b/ansible-hadoop/src/cluster.png deleted file mode 100644 index 590d28613488028dfa2cf30a17305b051a79c13a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25989 zcmdsg2Ut_xmNrD92%@MUiVCq%l@6jH5JaR&uhJ0^LJ0_=g^nUN1eKy9MX8}GoltFn zAVos2HWWe?2)+FKK!SereKT|K{C8&Vo#(j^=Ipc2K6|gd_FC(G*E-+5rJ<(4K*vf) zMMcG+qTz9Q4S6k9KsjG_yxf&ZUZ494v}-<*F_ZC%@h3V zdHR&uDN!EqOV-)h!CK$?vI+_V^%oK67dZtzqN1pydXYm|4*YgN*;|8u6s#@nouEgo zTr52JQBI({@ELv)eyF>gt&5W*=yX9yRFGegUtC1w48O4G8F1&h>u#}~&&l_}e zLs?n7Zahowh;edqz@VJ}*l6j5MuYACLq`i27blNDcDv%_KwcbqFlVstKRg6AX#Yv1 zP^&)53S+y`_p}&!dQiKfHOj`8@@64H@*PJDN@sF|tF48VlgGyM>rKCJ%7z#z!gO(R z0^|L*XTRU3baM2Rvvz=FO_3w{JwR{%{%1F(;j(#3{(pu$W9fnMa&|R9D|@QRT@kle zu(j0S0|x=xcx$UaN&MGK$@6e^a&xgH@5^6~Md^p}qUb&}tTy@5$pvHUWaEUkaFGAi zdd|hk4Q*u&Z9g~!=YI9M=;Y)K8X*&4XN|#lk(JQG4dVowY%z`wyBRd1()9Y&$79{%uA*lb~rk1~t zyzv`y`Tu78Q>XqRE+{Npc_X%LeKT*=fC+c>D_8HjNI&C<*`3Xb4k`62%1m?|V@3jpO+b z@t{D;kf495yf8HEzjolXzJdP*P5CDeoXA_{Z1Lytc+(yW|8E+0H3&WaPaF2X+t^*U zu(Y>XkNx>9ogAD1E{04x#>D~+hT0G1{$8mGx|C`UrUmn^2=67zydB6E-N>u&t z#B`L~Kq~7&_wQ!}?Chqf{wjX_CyB&PK~XYvJ}K}}l*}Fe^F(6g3H_^_5paGtH~zm_ zU;ow5i0E&k`WHGQ!e{>J&dBNiGiZemUWh8;ujPWr!@RMDq-!3!WjI@AAl=} zREDw;(4Ai+K^Krm{AW_5zePnUVW7&EDzbpGf8x+6pz zT5Cl$^tI0(51*Z5(Y)~>Vv#3wZF=>J5Q)6M60Z0$~;2IiE}0=azi49b=nT zvnLEoUrVpbQBl({BG>;^*}$0ZuB~5uIaxKn{-cf;8E&X2Td1(v?Ph5ewX9vQig^ev zB7h3E@h3tf08wz=+zRTZe58tVy_JIrp~!b+f7kAf;lZsv0vbNyPv#{wVC3hZ5f*4@ zwWhBMUISwtR|wErf9>sk;L+}of?ux~L}=Vb6dbZsy8o*U#eoceq8AcVeKa7z37)ne z8Wbr_9n|V`=)!OV4_M7 zWkG=fbP?L&tQvgjp_Cm#gfjs_BnfKM0;z=4P~V0&;kRXzSEMChB_L~;?%A@Fq6Jie zr)e3Js;TdrKq6daO3;IZ;|!Eac80QX;kmF11}$y9r#FslY5~&;pe!eMAAsf< zu{+@i^q@J|-7T(ZY22F+O2b}l*CPCQ)T~8$C1MPWPFOug1ty=AN971<(K1jxV#hJL za`GAx$u#YmmmX@3DPb&Ohmts-+*mnAdGXJ^8sXg;`U3S04N~T*@>!Ay*C!`|h#qcH z6#>c@_=u+VIu&gD-rBl4VR+(?IPYw?(aw*m_NIiI!yZrh%qn^#xGb1yJMV4R+Tvc7 z#!sFqjXpwBvPai%b$Q{G_pF2Wc*&1m??_(V(MHJ$%!p@~K)A9jadqF%+Z374>ma3) z8=i>eHN=Sb6gu|wmZlq4`uSot;k)nCY8?}NbRH~>@#a2rHfowLe1ca#?60q_rJaZ} zZ55QS*#1Ygy~V8IWcRf?^i6Q{9oCdu@%t^Zb`?>=&?#A0?fLUnBdWq}acDT3YS0lTjfSiNBnAa8+o@{6S&6YkawY-c--zV*lZuX>GizD$haf3E_Rz_AbS#?ke* z-|NTWB|`kFRa?4lo(g}-TldjU&F5neA3sKjJ?@SOCNH^S&m$qr8)H31P5s^zQl*Q; z49BVf>e*h`CT63WxdH!yD~(StIaCboXv@$qI^+7iGhH|1xju)FUsIe^dzO)S>E{~< zNACz!yuiDSbl2WLw$~Jmq6aoK%h*$rlX8THU zuSvRx6uvt7W*0i9HaeJ|#SnBi$r;1C^Db2SlpjNi}TVnW5}5{pmWmFIWV#qV&(VdxO89 zNZT&yF`i#9P}$3LzG9|DWt)|zKnAQvyx;Rp)X7ViOWntW;RURw&E?)6rKENvM{vZm zSuoSz8_?eHmnIbw-|yoyvKOdcf>?bb#*a&`$! z6KG`9iaF(IJbNO$_%+j{-S;+qvw|xP zPfASn8g{XY*2Z~#f4l{&DDK#86?sBmFUnbB^u?RerHRU2O-raCgxIRWHL0T;h7pN` zi%OGXoGM-(3p?9VfPD4sJy-lF?1D+ReprgpZqgnV=`kav7~V98?tGkT+}TTcpOZ{A zNzPb$on0h6uROoLd1g#}gnCaI(2DGP&opZ7TGK3yiqHiU6-#OvZ*X>P>FBg`p0fRK zqxg(Q!NxYlOby5c>8u^zYX=N2Ey6i~ik?yNUI2g5iSX9s5P<##2v;@W{%I4J3!c^K0y zn#1|l@PlIhtP|#KrNorDi%$47pE@IhbnVo8CT?xI=%*Rc3YID)+Y|6-bf;6* zMFW04ZTj@m^PjdH6tj!qoKB|aFRqfx zRVKCF4q?OAF3yhGZ)5j)am6>9R_md&A~%d|e3p3+dXXZMx*=QQV*HosP zClGV(OJlK!HYbJai|gLx?NKBfP2I_aZF3n#Eh_lstT-K~8aiYBOalx2(;S711&%%K ze#<`<2yf<@_XB;dCYkPwZ0d>bXJ*2d=RDDG3B73Mzfu6qo=P77@J}7$ftAs`rry3q ztZ{^1T8DPe$OT?0P8D$vV7}BdMn8x&i881>lJv+PeouvEiUi8|oQvt~Y^UY;jTgH$ z!(QI5GaS#O!43M+rjZRK6ieg-+n%o|{GDgRHbC-BPy|uVQ7?mAP&Y;cDfg?U^uQv4PAPNq^7OvNIvTuBkLZ~w4D-2EzjVrSR{@A+_Isoc5X4GYU% z8o-YmnZgff0n3)99Lbv*B2GJqkih=&ZF@XI>}AF1VQ6wym9W;6w_vPy(k-^xcVvqN5sts!R?HpLO2vLQ z{oeYXAT?-2`y)Pyax2ucla4ErYOZp7@pV$PoKevN>=_tNlhWFI3ZxHKv)Go6)lvwEb^YX~Y#1IB{>0XY(?U#=#GJMgzlhyd`9N85J(?0e5 za;yEh{GCZQHqTGgEHix=g6b($%W;Bw+0H|!o6#kM`!bA*vX@Z$0(tEWhb3BhbvoJY zpb%iZe9lyIb^dFg0D4%zi0NjWYvY-eDvx)!nfp`5rSPMrh{#04`;1?b0L|G6cJr8$nQ!{lo8a<3^HYC zCf>k#=tQqFzetbHOIzfAk!f7wa=O01+*?&D7sSXSCfWXjk@^?kYTeE%44B zD$EJINOuu00d~LnX9v$U=a|Xp7l9RYZRdJwE@IP|-(l+A;J3Q)qs;i#5-{ao#l;48C4=ZLoPQ_m31unI!A1o zQ!2<9KrtXa=|wA>u=VX&|J8+doHkeK%52Zr>e56JmPCYN35WL&I8qSWO3_Ph z!xdM*R<|c9?JY94JXfG5n}kk!B4!`ebD`%%_)~GGIQq*4WtZRGMjPW46YKSwhQGak zs2XVZqcJw}ovQuikI&`N%ecCdsk##;<@sizk-kJ+Y6M64EreKjNT~8A6u^P-`bl?9 z(NO!#ee0GE{F%g2$)>CC+R?UtKxk8_3NBCu&O=DE=6?Bxg@$b1TTO8E?>%}o!2HV` zEFS?-FzhRmZ{5mj=?1tT2|(f?Wbv;00tL8G!EOP%K{W|IIj2fb4tRml07jIL#hp$b zI+coM)P}%xy@X!+{5;z)6O=^9n8X<=IUoEHMrlMG0671>M#_$j209jV1^^@w05(w3 z-60R3uuS9P;&L|vyOm?*{O+VEox95m@JE}3${x`B36+lfx@!tRUZQDn>lqz&IF?)? zn;GvE)M}7i`wQ@di$Z8!GgU=4e=kL3+@-+Zbun~p+0e;b%;pocNzXtVD;ws?U5a0v zHvw*LLV(#w%oZ3a9Q4 zKsLiK(`mUvGFsf~w<4I@Vg)ivIiLW#miA2|&FzGos@*@x+P!0FMM?tN* zEN|Etxo;t{KN{-lax=>!`swlkUS>oh`ryh;Nmp!a4)Tz81Wn0M$CtM*jSfHBIylW@ zFV)7WMfBCPlv=H5R2XEWeT2Lf)EZC_N+id^Kw309K(!LK@i(Nax>5{H1JEMEXaK=@ zW4W{RleYsZf$Ua%ebNq-O#vc23P=v+)p>2e)^H=T2Hz}hsCabyM#RaOLAAg*83|N8 zXXm!apD|VAeWiR%bj*)ts#b;;`2oYgDrU!2UfeqUBi17>)^BmR&}pDzQX05#2M|!~ zXsVI83J`$NK!vYmgPQclZ0d9Ysb;*ua~2KT-v1E-6bWhOA64FCKdl0z0Af(ocJz_b z4Q8&&49sy=f~}P^00!VAV73g4@XVbK^$Q*z~$aNV2TbX5DALS0rdXKKY$zZ$x z(pXV@p`+2#bo|DYIclLqECab7791a5oP6J0ma zkrRJnO~8NQj_>l%8vaVt%Y0^DN`PjnfaMTO1lB1xlGT(51B_W&Awgzm(8i!RD>eM$}GH@zOYnM4ob?W30pJ6}w63~74h3aHB|K*?ARG}(TbG@#R z$3(9HhdLfLVhtc+%BSgsYddpgR)2=g7FDjO-cgm?*6+XOt1XbaFD;q`08f#}fUkZY zJmny|_!SuG>Qa;c>O-d$V7DD?;(ZHOD%a-LdZ*%~yt{8RpBSxS_s@qQs03!noVOBK zXZO#+Ecmm@JOhrkcP41cgb3HR?P}C7`VsHnFX2A=$mx>smHl$DJFM%F2wzB3e;5#2 zglV~7OjkcAt^kR9-B>H#=zk&}&XtTF~fL}BhZ1P_u#E(w+ua%DVmJkNp&dlDE92YcUNHeRfkOK4o z+V<8I;6WN^E@@R8)Ch*0yZbDP1w`auj%gf`F1iYX@0h8Ikx?U zg&Yf`t!ZEqOauPtiNML=b4Lz}qo0D%hWF{<@^q7UXvjKB%7x$}{E%+Cj!@g!W1Bdk z$ebFza==>ftE1{Y4o-iOq=r9bz4Bb(NWaj%%%sM|U1ui)P_$JKI8+=KrUp}*f~Rcn z&KwCj^yS74+j*t6)fLANkA!9cOIdv67^_W|566KwANJebx({q98?0i7R{4!#s}2w- z5LDR#3FINQ)vyRNtR*>;nb&Jj260p--K|p)Rbd(c)TOiW^w%AwpMvNwz6xxY=Y%tV z^pLaPF<=pL_zK>KICxm-@^yv-!f#{TCZ1_HNU`kYy!e8kCfU%Is1U~Xc_n?E&Qx@# z=28dy_M)$4ELkVgOuRDY^&4RyEw$vHg)dq516Zp)Q5mEL4`yX2_E9DOe1^a1KlPrg z^k>?AOB$QTr^7)yju}lar1w@X^-bJ%wARzp^IXOWv%7u}Ld*1(dK9Io#&mbSv{aR{ zG)%3MkGE+X16CyCAp1yLR{o%U<#B(yq2<)+Z@_F!>1vceXRXhSbMvj9IdduRit1_g zJNx*0^QTt8;SdikYS)O^tA6(5%%F;#$$q)9o1Q!x0l~D#)v7vV(>q2Vs!6;zscwy} zTTJx+D9_TBr1Y@2uMX>sXHn$+LY$)o0I{`Uv_oO+Bxlrs ze1Xb{(%`X#n0u{qh8l}CGOM3?lLa&vA0(Czk#6$%sui4*4P;YFKKOXXYZ+Lsi}4~^ z+Rh7K?g-#bJN3h^|%hL((6EGdEF0dQA2{X;oSzz~> z3>@B0gU|E-R(t=H8&M}K*P!0xvDeS6fVmPKc#N3+_lmS3r9DsPTGmzu)=HY98^r+| zG+sI(J@>)P$rnV&eMyb+K#kL4+V6aCqt8J`3bpPz0>ZRhIBa(koqTyZ{^fc9W;fy4 zRYr)#GQqdJH(Y3yBgWdg-BR{XD(F>_?_B$zTh8-dO)7GnS$cbGZ&pGa zy@}pPOY$8Y8+dE$!*t0a=95EKISOmGX85-m0s4e@1u9;OKbaM0`c1tCnF)C+Bzdcv zdxuj5G=4mFReOHNdJT0lL7<9t&zDdR=67R71LHS*Y|rL3J`DCfqSYVtpjzMrs;5?` zcmI{x0i6`K(C=gVOSdv(ZWP;A0GE&=pv$`WWw31$u4;cUxi(kK#CV}Q3J_ec_R*dX zPv6@jKHy`CSCW^yM+~G3o*gjOO}xNBkZWv{OP70g7OUT4(n$Q=k{_L4ULAp*zV9J7 za)BDKj{~W~Ot#pCGLJTSrD?h-x8alcA4l;!+LXAAlY{EoEAm6@b_CO*qo=}yI1<>@ z7zUb)HOogHl(MC>XXnxCma6?l}!=WvqU$%yJ^K!VoyF+P`B#Oq|sECSLM)1cL4^NrmeO1k3E$#sk{{s<}e8^tlW*Vbo;xwMzg#t4K8;?-&z483y=dj zB{3SLJ671F2<##$?(yT{DV~JZ{aEpbb1qF%bC2eZQ4?vJBiTalw>KxfzOlnr$}(+w zavlWrS9ecZho_f500s^>rzvv)r=o9|vM-}XvuSFCkQ!F%y@nsj7U`Qza#IElQLL}I zCG(QV+(o5{;~8$s5?yxQeN{;+=0H>I%Y)=&i`{ApzFR7Nlwx5$m0+&V2OX`s_ z4PHLSvB;?(ZjSxL>Yx0w5dSQ_*!`W>uBnY+{*yR0@fs2Z_hCsQ#uQ`O%aZ~$rEvEd zS{pQpLa~5*E4?N?6;le6HF<&^$tt=it~7j~w_DFrx@`o9A|Y(OSL)Qi^)(x&D&*Mi z(6hKUWjS2Z>^EcXNSe3v$r#UUSEV-WFLE-QLpb&+CdXvAj$};bXQ#QNq&Da+2ywAM zF~Z>X9J7b`xqk1yCS~H|z`;u_b+gx~+{koK;M5+3VrY1WT!&cxr|i=4+Xi)3&*31q zy4QqeNg?3_%tx*MV(m`GPJyZuFd@1mop1^BDykS^YvsGDGJ-Ma>Xtz-tmdwKrEd?O zK0O4uu%LkRym=G3bdzWJG7vowDCO9zh(i=usd5#W(ac0_k@_+FnqFYW`vs9jn5elE zdI5Fp(cMTy7Kr=_i<3)M*ieto2IXA6A(DVy!SyK%Az{1m%Q3b)2 z8}LC_;!0)WVs)#a@R3j2AjuWa_6U;^1=ED5-XCix?3=00skQctNs%7rkWkdSOyL5m zeu$Re6VH;T3Y=mFh}O)1AR?Qct$7z0x4fS6w5js9sl$@)*FECz-?zY~jFov!UzE9? zP@nlZ@?ugF^4pK5rLu*eu}vstzvatl9S!N!{x!*Trw^gi`WNi31ps2rZAV!|Y5>2| za1G1zYuair2EBB+B@F|bQ2Dge07D`#1Ig?isqeRBY@Wx}#u1R)QibxPPrhRMxC>JykY=AUP-z0Xu5{}BqZ~gv~!(Sy(RbLLFgyfCl5j*FTep=k&+5UK3>*VAc=M}Yk411*FA+6qSk2ubrLccgXCeh?_hud;TP&-L2_^P0B;07^a-nt)`N z4Uw4*3Od-%&UaK#_++H?1S2;E7e1;q_uJ+B0?x4ObL{JbHu@@6udSk8l$rMV)A?r|UM zw*FcZCJqk)F6dt9VaawqK&IbcLwy(nP{Y!r_i9K(yvzWUOf2eikLd$PP_G>8wggzE z(b*m+ajB5JbHQ}(U~M7d9ft?EY}=V{gf4kUY|k2r^90C4rV~WOpJ9z(?@97M)!)k% z=lQ(A_T#yS{C#d6ChjYwj-Eb`0Y`z#eJrsdr`B}WCd6FL(WcCloF8TF7nFMNk$7JxC|r>Mo+$=%qoon$b9oC=EhXs|4Ov zW(|R~YX9^fIuk^ajw7>Ax{Wpgr1p!R$%WvLP|*puy&;CGj)A#YTUkAVy#d*tX3`Z? zzlA~mc^|OQK7$Sb14`;=ARAArUR@ekD+D0zEz=~w!HcRWeiLGK6<`8VQ?Qtc{yqq; zv$`i#XN$wCZ29So?#*R4&p7WER$G2vJd~Nr_k9a4d zZkwlmWb)69?jI36+PxACyC0qP%Ej>mhr<8D!BJrYY*(qnuI|!uL*)XN0}2JM7ix=V z+}$s9)a(F@na|aVg_g!|-@bJM{`gL>@mFf*9oz^_E$2NYvMAz|*&4|#`^Uwlu0qEc zo<|=s7D`^25YHKGm#tA&m}yPf8co`6*e%B9P@ntiUXMgsAI`Kv*6G`;WP;R7|dZ9*6MV#R7epLVU8#O~d z=_KsbJIgSsD^lLG7<~pz+YteJPzi-lg@0j7f^y2^oV!ejhwihSPyeAH?lH8}nzU-$ z8;1?~wgQNO224=PF(1)F_4=|TQw~0YbrT?2O4!ncEUSz6b^x62B=HgFp(JJ0e+qwh zSPG`dB@tRmsAw}v-kTAvJ^>}*d!k_%Iiyw<>Y1O~#S%$C(mg{T6HTOZqLsbMlFW_A zv={5mRxj5_3lu>)zrI7$0NR&0;J05`YEBFuuUzw6`u0%0S5?UFPMz~(Ki@N-ukY=R z9RLS%!R=N_{p)U1$M94(nHkUtkCg(s1%5M(zCn$u<-+R-q?#-56 zXU+T1c(x=drMTu+*pBpmkrI0?*CN5d{_%`%ed|JkLiS<6nV>sw-^k`>jmBf?nNDVSOoOrlkyvxL#M{+By}U@LkBrr;kVZN@GSc;P#%FP#43?V3 zv4fh2pL=toQxj_hwz8}-lS{9veOaNdhR)b`$*l+?c59Q2{pc;v& zk(gpwd0wMLedwUJ#$1TZ@~bSu9jpqoG4}n4Qk#STj;|jhJ8@qMM;8@o=BR55f6BWC zNU)YnuZbs`yK-S>k%#G4+V>51u-}7>+YgkT=`P=nQ_9bpSIhc^n=}@l_u#N;MRGm& zVVp>}-HHK7pbovE-dOx~55LxEvsiC8Q~q?t*gR6Ls6bew6rMLFLkxd3NvQe0dfw!e z*Tmz;kM}KH-yZv;%#E1e9Let{n1&{5G%sim%734png#^^@G-_Ygs#;6#rCZF0sQeC zr*Y!dR_mCxh4?k0*iJQx;p2p&g$IIW*x}^yajVc8R8l4Eqss0_R!TKVBml*7(W-CWW| znC#I$sLRs!)1z`J*8AjrtZG3h_7Zci-ND%2f|C-skXHKhgY%0cIi*Lo;d_=qpdpgg z@upMEPlejfVoWE2wcP1Dh?Jygp2ttUa2ZtgC@~964muZm!p-`^56;-InO54^hoTNj za+&!T_2Xxa>e%P>ff5lr#F!zSQ13D z%IMTc!QI|d!8b&Q@A!bs+GsvO&Gij^MA&uYC3vT_=ej{oeOY~tm|LM1fIox*sliKt z5bmq;hWjiI9)t(r&t|h-h)obZ{Qd|0Wp})q^Wv~#>^G*f#N(-sOf$se-yP>bjZREE zYBU9QrcL(EF40x(=+yks=%O?a_-l!ucEzt}h6yI-SqO2LgJbGkmj|_5l5@ZrCaeU7 zcc-~TnsH5Aytz>_n-Ov;`qhkrY|qexs3}h+OVx7Jf#DSD0@+t-Q|d9N8klAPz`#ml zcrKdJL?{f(7iw2}CJAimU-bk?ac0l{)6NOrG>;SX7ajK zMC?v$+vTf(s(P&&Sr8=6Fs-3piJs9Ah{6ukPiLb-zqU`Ch+)o)4-8+*#HT+qJwXZ& zK3GKq1=aUp`CW6+?)wg0=DmqL02j47=ECAvZf_W^Oj=fwf~mNT>4fZsOk4fiA`N5)KiU-kK2bFEzf>i0v~JPtP6;NNgN5KrRV&!cvc6FG0?0(>L#00KF~PIdcNLGx)) zyG%mvjY2T`P?PgJ$SQz$w)j!uq{)v$g~<<2Xgo)VadFis;%vfu!;FVjzUD<^@7sE{>qb@P1JLpyfoE&rC;Db`j`aal4Rh9BnOrbIO*2I= zsGkPgmrmQOD4&;lvrs6F=@HZ#prjEd31pssmPk)XCY91^^f zTEakw`L}gn25>vH;g&HaR~sM$*UHzU!SP_tQTr&{#ApR(hG3T>-a}5pZK+|E#as7M zxPbKs;o$xIYG0qDh~4e4i&gXH{`O*3xP#XGa}fV_%B(G@X8^S1*jxN^@~*UjTsZOC zH6$}7>+M5x<-;QjU`g`;rPBoAzk`D20Z{(zo!h{u4TDZWU@tWs_FwS=N2WdE`YDbzJ|#Z0`A3toOR(xNP#Lg$L0B-ddbSnP&E?$ z@pw=ObZ(kiX9QrRZYaALR14CZ!vQCW*1jO}919iif5D*=)d}(w7iF_Q1U zEKL6}0vv_oe^ixPS#f=URn(R-ZgvTjq7H;HlE~!;fk|{}`Xs0z`0m{uMy2jqBk~Nv z=ThS8K(_giMzq1SJo+>!aOtqjK#01ORHwQRx4hd4K)1M`t`rWGcV=e+S^$7kE_*^s zXuaP%X8lJn(}promkg@#MLO!!7i6z(?E=2tlC?TvA%QcfHG%C6F%*fXZHCkzaSKK_ zTCwo`pyX8TF%ujeCn#pLVi6>^RA5E>Zl^^XOrZvpyLDu(2`Q?MQ{Ub{Vc;U&w(CtV zLXw)3y_0JP>F#FeW_-^@#Q?bYu}z4H?h;6wIgGSt%>r7Zge2>GM3I4bKe3;DiWohp zP0;!XoY1qJ%2oVub^5zIETyLJU!x;#ioi?0>}R5qEOcu#_Npe%?02#At| zGs5vVj(bky>&?ERbSG*tAB1YI&mEN7eccNjze60vo8&5s?aT(1e&wp5dunW*PQZi2Uhn=(QWi){8?j}i6b>Jlp&l=t zLaG}8rQ};~x4CM{JzHrCZZj}&WT6Z8kvs{!Wq@2Q*@`nNZljheHY<-bT?DL^iF$g6 zJ3RY@UZ$W)W5=aVzSf-l?n#gN)Qz9%~?@N-LA{J z>G{z>Hz(`xL0jvrwaEdp09szF=K&RmdS<65ss7~nBHRXq! zwmWxw#9BM3Huc|;MoqE60mum8i^ny6wQG?r?66@S!D0Kx5`XF-GB# zxo&;0vMFONa7PVtpV&7B#&I?S5)jmkJpo@MadAKuAD5LNja4YOtl0^^R(SzsN197d(-kYv&+qJ&#sN48fjB)Z>Ky|!&v;So?KwcN={cG#C0o0C@sD&}9)s##VY zy|ck?LsDBW&I+8Tv$}Gn+9PC?ZxBdKdzF!ZBs)=L6$^|bokrH~R&gz*ylVJpf?17S zS8j4S_)H-Cz3cZ6FJD^LKwns>S~`HblRs3M_6-v1!HITgP19`8Hj!+=i~hOFRI1JV z?ake%JSWYgYrJpq8!ar&3|m4~1_wm0cy{$J0cO{s!`L+xwF0PYV$^Ul;IQ05X^_3} zuQEGi3!Iygnx;h!*@SikMR~=2P(=V#ii*Y$bH<3G9(ZfLstVjP7G;Ef34IU(9P1k| z64Nz6TpEI68@ViCD7qkO9aN;0dGlwoy-G&TJaugpf>(gEOY4+zYbbXDKf+(Sx;SFe zyZ|Eg9^jlJ8j=Ao(@CMPhoj5#HD2LhqZ-^vUC5Lbm%vPkNJ@1s%^t4i7Zz`PB=vKK z!;i>~cG<)i0QYAI5^GtYjD8l--1uRz-`A&s|L#fG%VLci$hE#R)|FQ?v!8!uC@@HQ z<zJTdJ@`wZCbl(@DshjBCD5mcp$v{Ti@+MI@4k_)}( z(Dtoq_jyjcKlWMF(kYu2@`Tby?p7Mncu=YcsPhMFkjWd z7e{E~A8SgD6YJXzTDR}zEOwLxP5}557eZaeCs9z%!-NEY^;*g^YSvvnc{b|yfagV> z4w!=T>Czeo;{+(iE_??Bv=9>3bxe|L%hOpvp6;MBtOrDMP9TcQJLTBj393yl%BDQv zpBriMh3d!vyhbm8FQdNoGoR;me!5XbTI4j)4@lh~K?Zy|Ac8!opI#^*ofSuT|Bx#N z)&?qzPuz)Ve+!DDM+w3K!ieA5Q@j%^tWSYv(ij&Xi)*!3a0+01IxszYCZcgmVIw%K9y30Z_w< zl`Q-XoP;BR0SQvJEFuRJP>tVPP`t^yWzQwZVHDH8Sg%(Pd;|=JWy;ZG6tRKw2AazZ zS`T&E>&RaKLG!Mn)mmVL?9Cu2^40o~!B9;l=NlJ_oq>wkx!uAI(PAm)Y*Yc^bojI$&)Vx7(gYU$_7&zfPT8VP*e# zIjEI(2Pe5M5~LaT?U%ylgHH|KfLJ@Ky&SiFhgSU(qc#N)BN$Iolb-A0wBG3&2HHu? zrR+(z;OH7j+-wgoiMkIopx;{X)NLN^+ zpEQ=^-tW*ld>VS3?BTZ2rLpL$Y}djC|3(z29@LjyBbQMT6 zI(zWjw{Su3oF=Nlsjjag8jbOR06EaCgBoMsuoMwOo zgyO*$30z`bUm||f;!Gqf4elLlIQ1`+U;|boEs4exEm>pUw*>Cs#+6^U;JXhrvUwQ$z7j<-(GX*%UN!ax-Fpfv)z3f{C}k|L;t;i xLwdia9T8TwwvCEvZH=);vl4XJ3U$~*#dA?+Z)0)9Q_w=CB(HWp`j6VPX diff --git a/ansible-hadoop/src/cluster.svg b/ansible-hadoop/src/cluster.svg new file mode 100644 index 0000000..cf6c98d --- /dev/null +++ b/ansible-hadoop/src/cluster.svg @@ -0,0 +1,4 @@ + + + +
Edge Node

178.236.25.103
192.168.10.53
Name Node

192.168.10.25
Data Node 0

192.168.10.23
Data Node 1

192.168.10.24
Deploy Node

(can not be = to Edge Node)
 \ No newline at end of file From d0c06802d0136968f8363fdebc0e6983e19ef825 Mon Sep 17 00:00:00 2001 From: Valery Bergman Date: Thu, 5 Mar 2026 03:40:20 +0300 Subject: [PATCH 3/4] Add task-2 main part --- README.md | 6 ++ ansible-hadoop/AUTOMATION.md | 46 +++------ ansible-hadoop/ansible/playbooks/hadoop.yml | 2 +- .../ansible/roles/hadoop/tasks/configure.yml | 37 ++------ .../ansible/roles/hadoop/tasks/main.yml | 4 + .../ansible/roles/hadoop/tasks/start.yml | 95 +++++++++++++++++++ .../roles/hadoop/templates/mapred-site.xml.j2 | 29 ++++++ .../roles/hadoop/templates/yarn-site.xml.j2 | 38 ++++++++ 8 files changed, 190 insertions(+), 67 deletions(-) create mode 100644 ansible-hadoop/ansible/roles/hadoop/tasks/start.yml create mode 100644 ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 create mode 100644 ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 diff --git a/README.md b/README.md index 2b73480..3ec040b 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,9 @@ - [INSTRUCTION.md](./ansible-hadoop/INSTRUCTION.md) - [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md) +## Task 2 + +Для второго задания мы доработали нашу ansible-автоматику. Инструкция для неё +всё так же лежит в [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md), там же +описано и получение доступа к эндпоинтам кластера. + diff --git a/ansible-hadoop/AUTOMATION.md b/ansible-hadoop/AUTOMATION.md index 6e08556..f33ca9e 100644 --- a/ansible-hadoop/AUTOMATION.md +++ b/ansible-hadoop/AUTOMATION.md @@ -13,8 +13,8 @@ cd inventory vim hosts.yml ``` -Также, находясь в той же директории, создайте вольты с паролем от -пользователя `ubuntu`: +Находясь в той же директории, создайте вольты с паролем от пользователя +`ubuntu`: ```bash ansible-vault create ./group_vars/nodes/vault.yml @@ -24,7 +24,7 @@ ansible-vault create ./group_vars/all/vault.yml ``` Теперь можно запускать плейбук, который настроит ВМ и установит на них -hdfs-кластер: +hadoop: ```bash ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml @@ -32,7 +32,8 @@ ansible-playbook -i hosts.yml --ask-vault-pass ../ansible/playbooks/hadoop.yml ## Доступ к веб-интерфейсам hadoop -Необходимые сервисы расположены на портах 9870, 8088 и 19888 нейм-ноды. Вы можете получить к ним доступ, подключившись к jump-ноде с помощью консольной команды: +Необходимые сервисы расположены на портах 9870, 8088 и 19888 нейм-ноды, для +доступа вам нужно подключиться к jump-ноде с помощью консольной команды: ```bash ssh \ @@ -42,40 +43,15 @@ ssh \ ubuntu@178.236.25.103 ``` -Однако для удобства можете настроить ssh-конфиг. - -### Настройка ssh-конфига +Пока туннель активен, эндпоинты кластера будут доступны по ссылам: -Автоматика использует ваш ключ только для подключения к jump-ноде, далее она -создаёт cluster-ключ, который разносит на все ВМ на стенде. Пользователю этот -ключ не передаётся, поэтому, для получения доступа к веб-интерфейсам у вас есть -два пути, которые будут описаны далее. +- **HDFS** -- [http://localhost:9870](http://localhost:9870) +- **Resource Manager** -- [http://localhost:8088](http://localhost:8088) +- **JobHistoryServer** -- [http://localhost:19888](http://localhost:19888) -**Первый вариант** -- настроить ssh-config для подключения на нейм-ноду через -джамп: - -```config -Host big-data-jn - HostName 178.236.25.103 - User ubuntu - IdentityFile ~/.ssh/id_rsa - -Host big-data-nn - HostName 192.168.10.25 - User ubuntu - IdentityFile ~/.ssh/id_rsa - LocalForward 9870 localhost:9870 - LocalForward 8088 localhost:8088 - LocalForward 19888 localhost:19888 - ProxyJump big-data-jn -``` - -Для этого будет необходимо внести в authorized_keys нейм-ноды ваш личный ключ. - -Далее -- `ssh big-data-nn`. +Однако для удобства можете настроить ssh-конфиг. -**Второй вариант** -- как на вебинаре, подключаться к jump-ноде и прокидывать -себе порты с интерфейса нейм-ноды: +### Настройка ssh-конфига ```config Host big-data-jn diff --git a/ansible-hadoop/ansible/playbooks/hadoop.yml b/ansible-hadoop/ansible/playbooks/hadoop.yml index 7624a1e..4ae4d8e 100644 --- a/ansible-hadoop/ansible/playbooks/hadoop.yml +++ b/ansible-hadoop/ansible/playbooks/hadoop.yml @@ -1,6 +1,6 @@ --- -- name: Test play +- name: Set up Hadoop hosts: "all" become: true roles: diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml index 6440e89..a3b2eef 100644 --- a/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/configure.yml @@ -3,6 +3,7 @@ - name: Get NameNode IP address set_fact: namenode_ip: "{{ hostvars[groups['name_nodes'][0]].ansible_default_ipv4.address | default(hostvars[groups['name_nodes'][0]].ansible_host) }}" + namenode_hostname: "{{ hostvars[groups['name_nodes'][0]].ansible_hostname }}" run_once: true when: groups['name_nodes'] is defined and groups['name_nodes'] | length > 0 tags: core_site @@ -14,7 +15,7 @@ when: groups['name_nodes'] is not defined or groups['name_nodes'] | length == 0 tags: core_site -- name: Configure SSH client settings for hadoop user +- name: Template hadoop configs template: src: "{{ item }}.j2" dest: "/home/hadoop/hadoop/etc/hadoop/{{ item }}" @@ -23,10 +24,14 @@ loop: - hdfs-site.xml - core-site.xml + - mapred-site.xml + - yarn-site.xml - workers tags: - core_site - hdfs_site + - mapred_site + - yarn_site - workers - name: Display workers file content @@ -108,33 +113,3 @@ msg: "{{ format_result }}" tags: format_namenode -- name: Check if HDFS is already running - command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report - register: hdfs_status - changed_when: false - failed_when: false - when: inventory_hostname == groups['name_nodes'][0] - tags: start_hdfs - -- name: Start HDFS cluster - command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh - args: - chdir: "/home/hadoop/hadoop" - when: - - inventory_hostname == groups['name_nodes'][0] - - hdfs_status.rc != 0 - tags: start_hdfs - -- name: Wait for HDFS services to start - wait_for: - port: "{{ item.port }}" - host: "{{ item.host }}" - delay: 5 - timeout: 60 - state: started - when: inventory_hostname == groups['name_nodes'][0] - loop: - - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode web UI - - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode RPC - tags: start_hdfs - diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml index 6615830..f375e74 100644 --- a/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/main.yml @@ -12,6 +12,10 @@ file: configure.yml tags: configure +- import_tasks: + file: start.yml + tags: start + - import_tasks: file: verify.yml tags: verify diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml new file mode 100644 index 0000000..6ead9ac --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml @@ -0,0 +1,95 @@ +--- + +# HDFS # + +- name: Check if HDFS is already running + command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report + register: hdfs_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_hdfs + +- name: Start HDFS cluster + command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-dfs.sh + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - hdfs_status.rc != 0 + tags: start_hdfs + +- name: Wait for HDFS services to start + wait_for: + port: "{{ item.port }}" + host: "{{ item.host }}" + delay: 5 + timeout: 60 + state: started + when: inventory_hostname == groups['name_nodes'][0] + loop: + - { port: 9870, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode web UI + - { port: 9000, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # NameNode RPC + tags: start_hdfs + +# YARN # + +- name: Check if YARN ResourceManager is already running + shell: sudo -u hadoop jps | grep -E 'ResourceManager|NodeManager' || true + register: yarn_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_yarn + +- name: Start YARN cluster + command: sudo -u hadoop /home/hadoop/hadoop/sbin/start-yarn.sh + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - yarn_status.stdout_lines | length == 0 + tags: start_yarn + +# - name: Wait for YARN ResourceManager to start +# wait_for: +# port: "{{ item.port }}" +# host: "{{ item.host }}" +# delay: 5 +# timeout: 60 +# state: started +# loop: +# - { port: 8088, host: "{{ groups['name_nodes'][0].split(',')[0] | default(ansible_default_ipv4.address | default(ansible_host)) }}" } # RM Web UI +# - { port: 8032, host: "{{ groups['name_nodes'][0].split(',')[0] | default(ansible_default_ipv4.address | default(ansible_host)) }}" } # RM RPC (опционально) +# when: inventory_hostname == groups['name_nodes'][0] +# tags: start_yarn + +# MapReduce JobHistoryServer # + +- name: Check if JobHistoryServer is already running + shell: sudo -u hadoop jps | grep JobHistoryServer || true + register: history_status + changed_when: false + failed_when: false + when: inventory_hostname == groups['name_nodes'][0] + tags: start_historyserver + +- name: Start JobHistoryServer + command: sudo -u hadoop /home/hadoop/hadoop/bin/mapred --daemon start historyserver + args: + chdir: "/home/hadoop/hadoop" + when: + - inventory_hostname == groups['name_nodes'][0] + - history_status.stdout_lines | length == 0 + tags: start_historyserver + +- name: Wait for JobHistoryServer to start + wait_for: + port: 19888 + host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" + delay: 5 + timeout: 60 + state: started + when: inventory_hostname == groups['name_nodes'][0] + tags: start_historyserver + diff --git a/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 new file mode 100644 index 0000000..c74022c --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/templates/mapred-site.xml.j2 @@ -0,0 +1,29 @@ + + + + + + + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + $HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/* + + + diff --git a/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 b/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 new file mode 100644 index 0000000..699ea2f --- /dev/null +++ b/ansible-hadoop/ansible/roles/hadoop/templates/yarn-site.xml.j2 @@ -0,0 +1,38 @@ + + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME + + + yarn.resourcemanager.hostname + {{ namenode_hostname }} + + + yarn.resourcemanager.address + {{ namenode_hostname }}:8032 + + + yarn.resourcemanager.resource-tracker.address + {{ namenode_hostname }}:8031 + + + From 5fe8b5c8e0010f60d1d5a1a50a309012b079a5be Mon Sep 17 00:00:00 2001 From: Valery Bergman Date: Thu, 5 Mar 2026 03:50:12 +0300 Subject: [PATCH 4/4] Finally fix all bugs --- README.md | 4 ++++ .../ansible/roles/hadoop/tasks/start.yml | 24 +++++++++---------- .../ansible/roles/hadoop/tasks/verify.yml | 10 ++++---- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 3ec040b..a0a7a3d 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,7 @@ всё так же лежит в [AUTOMATION.md](./ansible-hadoop/AUTOMATION.md), там же описано и получение доступа к эндпоинтам кластера. +Полностью изменения для второй задачи можно увидеть в [этом +PR](https://github.com/5krotov/BigDataCourse/pull/2) (их там много, но это +из-за того, что папка с ansible-кодом была переназвана и были пофикшены мелкие +баги). diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml index 6ead9ac..7dd5e78 100644 --- a/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/start.yml @@ -51,18 +51,18 @@ - yarn_status.stdout_lines | length == 0 tags: start_yarn -# - name: Wait for YARN ResourceManager to start -# wait_for: -# port: "{{ item.port }}" -# host: "{{ item.host }}" -# delay: 5 -# timeout: 60 -# state: started -# loop: -# - { port: 8088, host: "{{ groups['name_nodes'][0].split(',')[0] | default(ansible_default_ipv4.address | default(ansible_host)) }}" } # RM Web UI -# - { port: 8032, host: "{{ groups['name_nodes'][0].split(',')[0] | default(ansible_default_ipv4.address | default(ansible_host)) }}" } # RM RPC (опционально) -# when: inventory_hostname == groups['name_nodes'][0] -# tags: start_yarn +- name: Wait for YARN ResourceManager to start + wait_for: + port: "{{ item.port }}" + host: "{{ item.host }}" + delay: 5 + timeout: 60 + state: started + loop: + - { port: 8088, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # RM Web UI + - { port: 8032, host: "{{ ansible_default_ipv4.address | default(ansible_host) }}" } # RM RPC (опционально) + when: inventory_hostname == groups['name_nodes'][0] + tags: start_yarn # MapReduce JobHistoryServer # diff --git a/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml b/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml index acac1f3..d3cd583 100644 --- a/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml +++ b/ansible-hadoop/ansible/roles/hadoop/tasks/verify.yml @@ -5,19 +5,19 @@ jps | grep -E 'NameNode|DataNode|SecondaryNameNode' register: jps_output changed_when: false + when: inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes']) tags: verify_hdfs - name: Display HDFS processes debug: msg: "{{ inventory_hostname }}: {{ jps_output.stdout_lines }}" - when: jps_output.stdout_lines | length > 0 + when: + - inventory_hostname in groups['worker_nodes'] | default(groups['data_nodes']) + - jps_output.stdout_lines | length > 0 tags: verify_hdfs - name: Get HDFS report from NameNode - become: yes - become_user: hadoop - shell: | - /home/hadoop/hadoop/bin/hdfs dfsadmin -report + command: sudo -u hadoop /home/hadoop/hadoop/bin/hdfs dfsadmin -report register: hdfs_report changed_when: false when: inventory_hostname == groups['name_nodes'][0]