From b515c696799ac576bbd060a811ba2234506a2d38 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Tue, 31 Mar 2020 16:17:26 +0000 Subject: [PATCH 1/2] [SPARK-31312][SQL] Cache Class instance for the UDF instance in HiveFunctionWrapper This patch proposes to cache Class instance for the UDF instance in HiveFunctionWrapper to fix the case where Hive simple UDF is somehow transformed (expression is copied) and evaluated later with another classloader (for the case current thread context classloader is somehow changed). In this case, Spark throws CNFE as of now. It's only occurred for Hive simple UDF, as HiveFunctionWrapper caches the UDF instance whereas it doesn't do for `UDF` type. The comment says Spark has to create instance every time for UDF, so we cannot simply do the same. This patch caches Class instance instead, and switch current thread context classloader to which loads the Class instance. This patch extends the test boundary as well. We only tested with GenericUDTF for SPARK-26560, and this patch actually requires only UDF. But to avoid regression for other types as well, this patch adds all available types (UDF, GenericUDF, AbstractGenericUDAFResolver, UDAF, GenericUDTF) into the boundary of tests. Credit to cloud-fan as he discovered the problem and proposed the solution. Above section describes why it's a bug and how it's fixed. No. New UTs added. Closes #28079 from HeartSaVioR/SPARK-31312. Authored-by: Jungtaek Lim (HeartSaVioR) Signed-off-by: Wenchen Fan --- .../org/apache/spark/sql/hive/HiveShim.scala | 18 +- .../test/noclasspath/TestUDTF-spark-26560.jar | Bin 7462 -> 0 bytes .../src/test/noclasspath/hive-test-udfs.jar | Bin 0 -> 35660 bytes .../sql/hive/HiveUDFDynamicLoadSuite.scala | 190 ++++++++++++++++++ .../sql/hive/execution/SQLQuerySuite.scala | 47 ----- 5 files changed, 203 insertions(+), 52 deletions(-) delete mode 100644 sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar create mode 100644 sql/hive/src/test/noclasspath/hive-test-udfs.jar create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala index 11afe1af32809..585c9ed6c7235 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala @@ -119,9 +119,12 @@ private[hive] object HiveShim { * * @param functionClassName UDF class name * @param instance optional UDF instance which contains additional information (for macro) + * @param clazz optional class instance to create UDF instance */ - private[hive] case class HiveFunctionWrapper(var functionClassName: String, - private var instance: AnyRef = null) extends java.io.Externalizable { + private[hive] case class HiveFunctionWrapper( + var functionClassName: String, + private var instance: AnyRef = null, + private var clazz: Class[_ <: AnyRef] = null) extends java.io.Externalizable { // for Serialization def this() = this(null) @@ -207,8 +210,10 @@ private[hive] object HiveShim { in.readFully(functionInBytes) // deserialize the function object via Hive Utilities + clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName) + .asInstanceOf[Class[_ <: AnyRef]] instance = deserializePlan[AnyRef](new java.io.ByteArrayInputStream(functionInBytes), - Utils.getContextOrSparkClassLoader.loadClass(functionClassName)) + clazz) } } @@ -216,8 +221,11 @@ private[hive] object HiveShim { if (instance != null) { instance.asInstanceOf[UDFType] } else { - val func = Utils.getContextOrSparkClassLoader - .loadClass(functionClassName).newInstance.asInstanceOf[UDFType] + if (clazz == null) { + clazz = Utils.getContextOrSparkClassLoader.loadClass(functionClassName) + .asInstanceOf[Class[_ <: AnyRef]] + } + val func = clazz.getConstructor().newInstance().asInstanceOf[UDFType] if (!func.isInstanceOf[UDF]) { // We cache the function if it's no the Simple UDF, // as we always have to create new instance for Simple UDF diff --git a/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar b/sql/hive/src/test/noclasspath/TestUDTF-spark-26560.jar deleted file mode 100644 index b73b17d5c7880a56981288d0ae2041b0970b14f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7462 zcmbW6bzD^Y*2ahK?(UKfX=#S;?oNRjIz&1Iqy%XYDQN~lkPf9pngNmSPD!a3&+*)= z_gv38_ucz5Gy9L(-)F7$*?aB%TaUUD96T!E+Xi}-srK8!_b+$=IRNBl$El-@0)V4E zLNHA_K`{07K?A_RpTGbB@c4KCQ~%}JXy4Da09#nu+x$NvxIaVeEv!Kx@c)xV^j}%_ z4qpE>IHBU7>qGTxpcTl)!_C3+_ed-Mu5{r4C_Me&gFUT1Y=5WI{~cZCKf-4Fhjbes z3umy?_u{nwTIJhifDHJS0}gG(I0ifbV2lC)NdJ#2ayBkDZVpyDGTK0Ckf)0WFNc+r zg}eK2*WNEuEYy9dMW#a>po-fFsa$2WZ>e;K(!=%_epX^mmXg-eYKPsl?dNg%E+DTPpqR z^&Za@Hici|PZN8?H1Lm&h;n*h!gRpz$J&QChohRhD4TIc`;U?u*5n5Kcsy{Urn3}4~(Bd1dB3P3*D-_w^v`l9uX67M| zo*R zFYYrA>hw&JS&87;WJVQP*2?Is-0LaAaBMNz>pB*^g}iNeea^LwXgWw)FOVATgq(Kz^vQl@*$r}#J(ZMk)mx1?fFI8j*g>g1+iaQ{LsdSjJz{Fr_lFA zaH}5uOl{&kR(-;O9p~NWjtVQnXyX@;R$kZ1#d!^rOsEPv$?@-dK;;eeg`ww^jZTwc zhI%tqno><15)stXb>hzmpiwXU$8I7eTL6Vs_XZ>jv2;+V#spr!Xw730h2{h6U-{J{ z+qNImj|wS%;&J5)p}SZ&zc@xK$f3y|x{CV9dwi9_?2|cIBR`hTuqevrALor+lMx~q z>5anEEO(}zcPz3rQHyplKax~L*OD_i`8aDcOpXyaBK-;uFKJ~HpC!B7_bUmL@pPDmRNAgKGem8lX{Xq~kAWkofb9tci!amaHpDIwe)lm2WcBLWswY9n`W z7B;3;j9$ncl=Y(m*|vg;Zkd^EJ0TLKE{vG5{0Nv7qy?JjNu&8s3%F9`7Juf zjOC-ObkNO3T6nCsmC-n9Lm@tfwjX_0*lfgUulVj{P-JEJelR{S&}g`9bEKW#s9sY4 z(*qz&5kgO_|HnL8th2)w!=7-> z^f5w{k$2mBPFGcIT`1p5{Cl(i_NGub(bp#rQd@!xti8YL zsubN<*qN$XKg!OwngXp`ma&&J*NED?TXD*9zbJlXTJ$I|x2w$zo1SAH$h;mP*o zyl8{H0Qz;NUe)R4wecvwyH$M^>D-4FzhJces{H`Nn653w<`x4J_We|WvNt~7?o49k zRcNl>j~{uvpDeNmm<(&_^`BhN9z|lEW1yTna#U(ph&%{#bGhk!$oqvU1EoGmh0c&c zgu7h<$;ODuS}$tUgP;ocO|_**DYov(gtk?b8IdWzn302kehbOrBbw}Du#!&HfMmH& zpHwUC#l+L;>~qyLQZajZypl|YDJ6Is}l_XkvU6vC}V>J`DAr_kG? z>*g$X)`)C!c&ElUxP)Fte!bH)S`yASbd?InQMgu-Nc6kU6Braw41-7(3^=>n;Y=kf z$Uzv06y*-6G!>{+q?(QGV+-|GdL6GvYb!RJ>2GsCZCmuX`MhiJv18jDiLVpD=I$#U zS?rMG<+K6Hxh_eW9ck9+)d9{74sh?i6uI%HL8MbeQSDi8#=nVTGnZu*Y0%jaotcbX z9c0yga;-XY*l}4~ML*SGb4r?zQebR995%D;)hMU%rm`cW-p))KBt2S+?vHZ zB+2CkM2PTLz>M{CG+fCQpr?75W+*GuDDQ)${y_d^W#e|ktJG|hHPgu3VMeS*QjHV> zUwV1&Udjjb>aI2*eEQH;ToS~C6MHg2OyQ(_zsy!BY zTc|WRrR*Mi7kJ}~RUqH#7z*>w zqS9y@7@3Z}Fleun+Y%PCc~xy_0FxjpKijK6qg(ui{j7BiR9X8PSA1Ur|aGmB^`DW+8q@`a9G%jz_>4^{8O77}DB z6xSYGKqL+@v)qhvwb?>nC#mjFMfoP)j6lfG?+=SJH{i?fYoWiI94b$c5nXi> zAXAsZKoCV+mN-((r1;V^xg=?Fo;LZCG;m_MM+jSsf0~nYn8~=fPy-8}{}81MyqYeb zVEu`|7BwJH@q$(<8b)KjeL^M3S-=G#BI*M_CpjXUk>n>vU{BY1h-Zk?th~B6<(P;^ zAY=2Ha`|z7d)oc9C5NLU8>s2WrF2_U!pTeIeu(`2grErkG0{eNtP^wGDdD82j9B-e zk(oK+3Q(_4-TqZap*78A)6$eCel_bn+>Mej4DA*ok$?n?q@nI>S^_sDf#Qi(Y|Q)K z+WJz8rHMBvUgEl9wycQ{SAm>7#OX4Uw$8O=;2Ad$?Nx`Kl!MW+-uN(|626ii^RP_= zY0*v%HAIRdLx?rCh8MErcvps^#xR+{;+8+tHpa7Y|kp3=w{1KxcY6C?|fi`*xx2;s%?{GPF*SkXa_ zfPjlY7>%bCpSW?5L^V#!6U-U2&{f*SP@6uqf*`Ye6oT31Pd}~du#%N@%w0Ko2|{R5 z`YOm=YeIKCRSi|942&dR4398VIGW|06Yb%CHXma#!-2U8N|e!6>yIM}gL94CE@Lkq zq6d{`&0EG-BENCrNV)%2{8S1v0o6)a*k;&BSJH*yMwxn@>?}qbLR(+g-H>X{B?@!^ zuBcs+h;qu(owmY4U?_1*&P)rfoSE)u;^;4QY7fQ7d#= zPTibIH=^FQBJKF$ZS90NuO!7`I~a`IAR7vu#Lm*(WRq)aJ{H`gZ+*o> zdOBnM=xBja!(k^M61e^{t_wTlRMMN6Y34zr)1$Dr#G}l3MrZ6K($!g;gEn9qVj>Fa2u)$8I9WXP;D$ST$$4rT zS-xF@o89Hg)N{%9bY-DxwhLA~0z)+vv1EdiG?tBVgBn_9g{&{OmZHTtJO`2+$X%(Q z?&G=7b=vJJb|CVQGEmfMT_n$ODomliN9KQwH=ZonW4&$tQbD`H&huOh>5D?bs_z2| z2^15itd3_5Dg+1Gt<_y6&utd`rjq;55besfsccYjkT(ER2Gqm4oQDaK8=08v?!#Pk z)#|tz!EX z0(fXkcq=^~YF#Uw+=^XZ0BR4JPGO^?6P$Clj52_5?>0mvo;wzuJc8IfqNYz~?A}It z8q%jj!JAK9YO_XJ_cgZeJ*gXoVEU>yGpdPOG>U1Y&0Vjkl8tjPuNq4gwcr3Z6 zOcG0m?6nGNOG*qxMMt7n#v0NT>9Ohui+mM=lcq}*2;$1g?D}L1x#JkE(m$1ud&1&H z`YR2@wV^DS-mjJrAfgg3&Wp<`qO{1@w7OKntwDPxI4?+E)9aM>JhnQA+=QVhUW5|M zEp5yBX!A?pScY$gtfMW=JKD5H-Nn6W6Y^4;kVp*%vv_;ag+-hZIVziV-j8r{bs;3dP;E(m+@9Wgi!6LWycQ`vj2J6j zP{0o@od@oM;vu;r2Pi2XZY=l#V@oG|j&Qn&a~hp6Ev%Mf!AYN!E}VLc76i;N29p7G z<1f~i6h$1f4@T<>sh^y7oghf>gk%Dwe)OjHjT+_x-z_3 z%hoRsAPh9CtPI+-Z-ohXzgeurG!n+D!_TNfBIB+}0^S0saHL0*(mq11Hvf3JdVRSK zTi)u|VBFGVRC(ifJ$ACyiY;z3-*SypYkURqZ)?6ym)e*;3h-=Mn?xcs?cQC&BtKoe zUQ(S2c-z!sVCMd2b-EJ~L@04f$q)6|mJr~bAy9pXAZn)qxn7)UQT2uFwgYc}^;xQrKPF?B?EagXjR=mRRDEBB z;TJB{5A+fJp~Db7k1$mcvNvIwY{)b@L7nipV6eawG z+D}mTCu*kcbPf0Gw zW_|5!-)5+xbL6z(VHqw~lf8p!x?l8poACujKuDg`SJS0rY@v=P(UY12`KylfnSMLC z4KHbZ)>N>Y5!x39c00&ek%B{ApTJ-HV6WS}w`eS<3xP=l8ksQn# zvniIN*oOx-zx_f>u*JqAwwKVTHgn_Cp2P+@)Lb+&C~`?RC153ITz z>4!>f|0g*;xs=zK_}-K)&IS(2uEAh6|!;oULu6Fp^fG zhxt=hOz$hK`G}`@@v;WxK(!t!USzmJ^=*VNX)13?gZVm`HCBhf6`au3Ix(_fIoF-q&j4cx=h zh#6}*TwCY^3J=4uCpeMzU5UkXKjjCS2nJQ@>FMhYwIIs1ak_vO6IaFObI}|O0Z;r6 zsyJ7w_7*_}AH?PuI_9s5IGC><;*U8<AA^A`NO=RsVCa7G9 zc+_Tz6Y2a?=Ss>iG)Jz!ylr$@Oun?-E+axMu?_nk$3gJ<;OgobGwm@QGQ4r?)w$@f z&+G+%axOPeyFifE^V(K=j7NxMDqJE(8u@If#qBHQ+e^pC*SOk63}&6fFP128vU%~7 zV#ihI!JSH>SHZ7{&u``1;0_sUDxPN)V~1qfW4uQvub7U*21}sivU&7?PA?o3mbUkf zjlU4x!u-5!pJO3F^tl^r6x@BVeq6Szs%hUX-2ZCT{`g}xY9|+V(BgH@63_&9wJ)ApGuF{K>8OH3<#yBT4K#>0j=}{|?0c2^jh_@Lx{G z|IQ`)i97UPx!+xlzlZ`nU6Or})kJ_^*n;@;&}gyt~JOAHe=?0e{Qw55-@) zAb+3wpU0km1oG|9=s^VhI0U`JeH()QOw>KT|C{Y?-9A0OHOx&?$kf|%+V+Su6sj>Z15 znSZmP-!=vTWc-1Zdnp)I{M%m9fPgUnuUIsK3OWjoHugsUxJsT6P=o(|m1oIq;yt`r zo$pD)ZK}}_Uy!lBU}>_~nmfVNIUyh^oXXab5H0eGipSOqkR#!Cf=|vxJ;>sAquQ|? z{R${L+nklZ03W!BeHqlESrq~L7SEv!&kG6!wHga-kb~xppD!89KB~RQthEtd$`dPsZ>DoRpOb-GeDrP?09%@Y?FL$0(cvu{Cj4o9LD z2H)DxT;;L*l8Uvuww=wv;M43F!-}VNOp{H!9!`f-Z3Qik%eFyVP7buWffLqBj?BG> zVef&OZea>NBp&))Wvs%q)Gpx&m7-}yhhN^sila4W8Et%-cMlKHZuOW0;86J|8bO6g z8EGpmFF^x^2T3aHCj3=5r99gG8qY?KcbyJi7Z}304ZL`8qlm2KQ>Tt;_;3)$(T3i- zrZ8N)B*`>6dy44DW0M%}gHm)^{)tma<+I~ z2$YbTe?Q*6>EOjnHVD?7qBKf=%_3B@P{XQdhw6NY09ucux2Z-tkbn3GNT2Y>`%tp} z6^<|7e~cv5|LcF>)Xe#xq3L$?Z^P+#b{zZ(5cl5$8rWDn+MDVB9;b4D1#J5#+7bOV zu#=&q@$cwFe??2dqCb+F>TjYOx$0TjTK@Bo)A{dL`JHROT?0Hp;4BOX5RfJ$5Rkxs zxQd98wUNDyNYn%;JgQIWFyuINriw0>^sA8@9Q}^oxuO(G~LNT z-J;NEW2E#d-bb5W(hR__K~n@3@KJl#C$6vwvGucYyJb-PqIs_5O7?Z#eY6N0oqjaQ zuWJ7i%*DmOrl*u|($;ym!gjS&C1=%syJXq97g5{BwOb9P9U{M_sa8&Nc6v1hY@D}f7o1P=;uS9B zuF0~rRG+HAG+l|jFRGKzadD)dLWxGkfEGb#EA&Ha%-O8z#|W8^Aj^UutD}&d`Gbhg zNf;#ci0wHJel-mRLoCvv?*(`$DRnoUbNUF6>#R+;Q=;H*Aj1(E6vvK2>QKe`Kmm+- zo7Q>XE$e$#V^j4b-3gfXI8FmgQm8Xrsc;#A#u>zlOkmfS`ZGFLtfd&ZH#B&De9JRU zCcQ1{a5U9=TTEi;&uE5dhvDnpaDmjTT`EjbTH?GkvA@WnjaI)_>FXg|fVJ1#!k(Zo zR}IoG8cC( z)T7-g$Y;-4t;ytTE^wLD-ebYY)_V-(yWO!KHvk@F<{u+u=gp|$dzQP(4NyWgvpjaI z>qR1+C(-AnSuI7FFC6DY)l}K%9&ws27yQ*#Rx9QCTWEL#@mA`2Vv*-UvOH(rg815i z@+&FE`0`~Ikh3WI@1XW+MA*C%`TQf&cE5E_wM|E`a7z=X!=y947WzGR^BEd4{Fgk+ z+T^SK+v0z!;4&06Y!LX6-oO&&k8*28>(tNc@2YES4Q_?}=*>cj1mzLyTDPk>0_dy4 zJn(<1J?BQi{+2$kM)_sG_xDQOYqql;q_G{0zCJ#kaeO0njTb<1aai=t?{ogbu{gJ_ zjq*_A+F5n0=|9tnbhmAQ(^T94bPc2fA2L=90 z>1-dI9-)=CmBz+3=5#o_8Hj?m6D?Vnj7Iy2j`LSs%l?2OuM-aWgh2B>sP1_=E_h@a zcrUEaXE-ZIbRCb05Sm6 zEMTW0 z%0wF-2X-Jb3%xhv9GbYZRhQ$>2x51_X9>Tf%;?dW0Wrqul(tIUtRYdofL(cD>uw^m z$j{LOO4GOIvdKgpxrazt20Gn$`tuMy~Hgg zLtqG))1eKGLtJQyX{16V;$ynqW8FTph@I5fM7vP-T3H4*-a!Axuiurv#mh%FioW^S z;v+h}KBANNKa0-)BlNC>KW}u*2COO?hf}$gmELlOq67>{A(yy@>2c~5F%s3+k)jurQp9_-Ws^%uF?lr)Dc4cH zHq#9s=Zy%i%E9*-E2Y&FHg}(i77}unw@8cJaey$tc>Hw^f>>#d_WFZL2ZvXjxQX|IY(DXOn|Yw zhxLJ_{;qeFjoMEU?SxD&zUPK^tlBFc6hHo}X4>sQ0cG0B*>D$QuFlpz2||cB(s)@w z1V+3bUPA?dh-4VGP7!fEweC!a+gH$ud-)mtusz&$HD)!bJFlt4^v5NRn_4!wSSD5*m&FL9|I zsW7@58?0ZUkt0A!RuJ=@M+tCZB4Xf{XPmLw1{YJS8Q3&q9R1cm3>jDP1vsq>a#XI& zGc^;^JT6yEo2PHM$ry7aIXlaI0SzsISn7qyOa`1ix;e=xi|*bIXmBzUAF@OZflV`e zW)@EPnqUZoM$g?aS5!1is@!ORhwy4EDP;FsHFNd^EIwf~msUTZEefBbQ>Tk+$cTMR z59ve` zwEmTh{60s$%f9isl%RRsC|Um=mwb7dgif7J8+=M!=#8VOS`eG#7k zgRp@3mq0LN0gOO9`MzMyWQNW7P&s4+ig7wJ>BLI9DuBRjiv94Lfck2FpTAP|>j+@W z&^)u7WDT@ylEEsWSF&#i@`7YyY1b^v`DQiqAd`5(sc;*$oH&q~-^4QMMPX#M!gk{h z?baec_L9V^y_Sf~(0Ta|iiGyYQ!ZJkeY4r9#*`YG@gmW4VmKuM)O!Mna+Q3gyVdDV z+_;39^w#v9_LXTbu^C-)1Qq8&Y*hD}g^g+f;JkiMrDHB#SY*75|@XX;?xHxd?3{FK2Vj;ppiqA7y8 z>~uSrgAR#JZ8~R%a-f*LVD||X{n=Rsk9>#S-A)!%De+e&Oio;kxKvsa)V`&)+-@;T zkB{h8CrQ=dLXn2QE#G4o*AI@=#ayE%9ij%pN*kh-p}`Gt20|N)XI%46`oSbY7G(VA zL(L<%fZeIsh}f+0^!_jE`9loV^ah_tiwJJkLL~vEhT;h3S>Y4&qlEYJ{;@TtKwE-IhHt+Nze3I0rt%x`eEOpT z&YUAMUVivW3m*!|_dg@nU(+aGQOjnA4v9;9B_Of8nV!4fq#vpl0s$X=vR8i(6$TL@-V0hkctr3#DqwSXAmcJ z!FRfAN((t&>_zF}`wrV#^^kR40|W2_#u@9V)1LCDlp3~`r`y^)Hzu1_9-Fi1y3EmS ze)Xd8UK~M%Z&A(L5W1&xFD@oq&rWGb{+w{2)73ty3$uc>fchgW4e3PUll2y}9;6FO z3HBWF~>YB_Y5G*2RZbm0}AHz>%#Vt5({4#`jGHv2AL~X)W=|;Xq-aFJkhMEwp zcX7{04o~~Y;d1}8Q2Q;1i&_6?8C*jKQ4T3!!7S(tbv_;*nJXWN6}TZ%>qiEM5K91t zLqAz@}d@#I6bfGmZVLbKL@@pmd*A879`m{sE zp{%t;X`$x|QaGsu@kExL!E)*_$RoFfk+f>68qg7{!Ss{FG2b?;6t#w359j`}-CfF= z<(R%|gOt)Jobj>weh?WzRyJvIDJS${y^Y&+VNA@U4U*-;ZgXMg8D+x|NGWnfr|ga5 z_j@B9W1T{yCT%m)giz-YeTnHCt0M@&0OFV!=a=t^xqEw%kSut>F-8sBM&)c8Qy$Xs zobBWdMo5K|$U+_F%kW{jpEKlEyREQYavQ@ufc?g9KQt`#Q~FF-N#^T-!Dc2|QURfa zn*)7~nUNcTfe)PP4xELN#V8>^oz2r++JFW+M{a(;j&5TSjim%mS`tkkkzI6_mo5QZ4`C6i8>%o_+cAJgo z)*0SzQ+ec!mwjoTP&(F0gI^VyoV?G;5TwonrF5)!2h7d$UL776xr~0P!o~a*>3;>e z?BR%+2W~2-J6)&ch#n5EbjrvNu<#g#Rjzwh!@mr#Ul7~CyJB0S)!j5wYG~-MI4PNO z)6BjWiu?)!rjG?{4{o82BC5pwV$SXOGi*`i@I?F#t{fMmI>%i7T-D6W2@-T3-qfSa zAee`S0iY#oyjmZbW1 z3MvpJ8iZ)7;pn3z@+e-R{@$_4LwA*dio*mi%5*ga?h~3?6td8ZHc^|#FKZrNFviBn zap1Y(GM?ggx3dHdl-CF+fJJH$yrmc~DV|yjMIA_D_yawDzZQTdZztSW6$%g5m$^>~ z;8LhinRE-m6v{z;$~{SRoS8f53(YbLUfgjW(lOF?F?ASnlzfL0U`^1*`rmS5wO}AB$sT!Wk3yRS z+BfY8eKV@W+;lN>MorxCoD-rn$Gd6JYUZ#zGSPn~qHISf!u4#3EO7^KKR~kO^fikCn zWHLg%feNI6gl7tA73>)#R%`)iIb<#V&}18FQaG-ZF2Dqll~jd`wQQTSXY}2Q#EEZs zhWZ@oHtmU6PgV!~VYAT&zM9UwY6T)9O-6;%ywN$tLq9Vx`wZSQLj9D=U(>T-xU zFBnjp$?yhN-WEY>w2VWNiCxpr*(3496WbUoys8S!*{6Z-A~KI$P_34MVH2y=2G!5>?F# z-eYXO1pW001+zitd!YoA@imjzPxCHT8H1)=OrV3R0#^iBi^W%)*dkT29!v>=r@~=) zefZtkYY0e#pai1NfdVFIOSNq=Yz|(oMEE)HX&?zN`m!#*a17+cr)D9n@ zD3spGcTEo!&R2LYd9FK&fLz` z7%E7SJ4}@hBeNN;6XX%YRx9j(>o`h3I*xuT?P7?Mlnnbe-=Pss6(r-B1FZ&2lyV`x z{ngkDHMN@50~e{Ey&2WiqiHFWafn?hRYC4Pq@su|isRYp?U#)*_QJ6oF^c|4r~(Ch zsh>uvC$5d4kB}n1H39ru-be**(kC1q3f!8wBb2ANREc95Mz}V%nR{#*>MV4Z@dSF| zwZGFDz(|5wjhDN#(VK}?M_LhLsBVU74&c-t9VQn1x$hX&8jgw1V|TaZ1-;|E@o5kh zh->~w-*I>9J2=Jv;_^W?#OguNsx*i#-%E>$q)cahM61*nA#slDIg!|u@K)tm^ryj@ z_eqdQe(!I$?eMv{QYr1j(O67RWP4_x4Ohs;<4kr9`q`6z22RGQn}IKZbQ|jMby+ZN z!Q9LL0Pj2O^}d$0{`ti*q-0NPRkV!2lB`s=UZ70?nem8&TJR2BzNGy^~qgwQM*pXF6LGto4S+BnB$&D}8L5*qD7#{PcrSEO1Mx5RR2&x5yt--|4fT@} z1yNn(oX4-o1BctLc@tC1&WDxNO^f0N>YrvGsLk_W>ZY321J^s|9(QCg1X$%}>=9lD zEfkCNqQwn5{&B+oFOsb?NPM06AldT&PDig`YGePO)r0@)=$C0WAfUX%rmb_rsB>lM z@Ks|mph%R$*QHi|YOS8lpCxW4@N`r9d}7fwB`{^Wnuh+(G_EHH5dfH{j*hZ%-K0OH zv%TybB-sElsD=^*7L(-$i2yI?jls7( zI1PMA4L>lXN^zQLnScu?ttYdbD~1SsbE=_XR(BR_jB*Kt`gFxJeskkc9#d&>ahpPxE-fv z&Spu;{uyk+n!6x`a=^g^^_b%Ze0gUZMw1oWd5$Z0 z*5FK4ZgE;L04U5e(@h9UO6&n<(q6$SV z@U`U}0-Z4qT}<9Z$*$|E%HR+XbqT8650d!EK64YkCcX}HYe75@^%o0nG9O7JatjQy z`A!gqfbv|KVy=r($t2mP$m`_P_;w2NWK_ySujn%eQv^iT1r%ArdgO8cxHqBI}!z-lGT3Ej7xndd1-7Q?vA1#+09le{Y%oXcQ* zZalJR3b?5iA*U1ng(a~2r%p@6Ogrwk+A#NnTk`+&+ECKwKdTJmBz|kP2HqPooDUtz zARwUHql3=Ah5O2cl7Qg+_)(ywI!}xDKzvpMVKu8$7^vP^|^gLo>Q~|$Ge3%~AVtql30sw2i zzG7c;DC38Eji~&tUL5(G_~WmxI*fa zy{5KKFNtNRLIk7_A3&MkgsXv0Ys)zWJaZyj$4P5TX35h{O}1Ba{sx+a_Ua#Y*`XV8 zjj57<%EM}BgT1j^%*NL~C>Qa1Iveej^puK>5YFWe#H@l*`LBNFpxpptFK+Y(Vx9w$)gp4P)dImtnObg!209Aqu`G9N!)A^q!>Ul(C!ahg)`G z^_yF^@ZpwKnmyx+^V06p3c4WSyBC$->I4`Xm_@~Au}fQUsF^LJpV1p2jxO>Z8howw zn+NHxP0Y#mCE@J!uD(^LW4DWTCMi?~*UB}0qOtCshi8s=BIA&Yg6lGnzc<9BSm&KO zJuWIlLe2GZJ>-z@CndX>vIbb;5p?MRSLTl4`KW#{OWfKhV!LNfbM3<|3peB;eTF7z zC%UIZbJZGKQ8QS$8J^L@${7_oD#M$N%f6vM)|DE;dipV5k@_W}oZt%i*eb;vQ3@Bs zEN`xgWkvj?q%itJwOob!?A)goiX2+mv7Z%DP&F(ZO9S0oq^xRlnAKHACf^${HqHv@ z{lAwa@?m9bN~> z^`EBCNd6MnH!v&W^q{-{u$^k79j&5lvg7spf znL%GahMsWXX9xal1K)%n_#!a;Pa&c4Zo`xN^Y}Tnr7f{^3-U3Z34z$?4ML>4hB^6M z@K~X$Mkl~jDedBxs{~Z-(J|o(4+TGKT%rSEGWJ2G8TDOR;KT?JwckX~9=W%Q@(nhS zKX2XTub^s`gInvs}iKL%TtghVYoGIZJdP^oJgvjO4F}TbVDZ)Bpc-YGr zHIwXGZ}^Y~`c}QNM>I+>=jDhi_9F`|L*zO3GP69k(793l!ZOPOBp2Tu;AFxIMMLH7 zJcD44RD>Je%%fxxuoY#CH@>8Cg+r}+DR00Dx9;{pL)g5}nAF*&O}=4vWc2oSGa9sK z<+o*qLrN6xTUsLAzh~r|4N7w{bmtunLBJMP2Oz7yI(#;99nB6u{YkHZ=V`=SWVd@Q zsWz4&lO5%@yye37d8l74GDGgu_!htOR#kAA&`n$JQmPBLWbJ&vdU@_#cYfIz7UHM^)En#lBE z9=yM!*bKES-eB|{y_ggIlF4G%3As!|Fw{a7D;EKzZEfJi8Z@$6o?Ns4v zJQ4ekd&F<7w3|je>{n)=vfzKlRX$O1lWrq>X7aSF`_TqJ614Z?wrhq}Zq{M6j4J znIIQdmSU)a25_*mTK3G|T5=ccJt3qPbC(!>y-@?RA@j`aa?zBD&4IC_E{#NCP$;oe z{wZk_dICx8rA^QDcG`NI!iaGNM+<%?u1-T#xEC)~;K@_mhJ6wvS>%Nk^9^^(w3q+p zVR9cCFivsg%a7%3i!MTAJxy4O4ZptcZO<2&ebXbeN_S_4tbYL9u(*CyN&Y~f_Y4*Nbq%4`GmWFptUBuzB)^-RJ*ir9tot?$AQBezm- zLEh2Ikb>+anQSS!Xnd@1VMz+chD<>`^1h}vuo3@N>rvi;@bP(!bB z)o2ihcU7#}I);sxqq{3_kOCc~*;nE#)rC`8mITU+G8aYZXoJj2Ob~!=+IiP-!=}i% zoqh3DOXzb<7~)~=)n)R@ly)1(S7JMl{rt=cv61E*z_F`xrVYm3#XK;S+j93K>%%Ql zG}W>5F{F%c+r6CaZ!SQHlWK--Tujeym$9YUvLjyR8C2-%YHQB1U?2q**4?KI4kUcZ zG)s<7!*eFoHmFXzOX;0FMknj(?U1BiMC(802LIYmXmFoE%ong6ky@ zM9)>;TRwZ%6Pm7a=XkK3PLHHd&DHuu4-CDHS=G7Rn@FB8#&Jh)tn%Y3zs1>ca7|p& z!G>vxeXT?{pFYC~Ca#HY*wpUS_J{wp=@*o>&UAg53G^D33X4}2^niOyv&0b_B2Z;>BV@$MT{w~dY=b}x?A58uqVUO5Rxvj?NtCGp_Lveg#a zf2g&=QKaQ-V-K%{E7+VHR+Je`#g|;BBRB^98dKEpfYS>)yYBCp!R$^Q&>{-B#817_ z;goDXG_O&6?Lif{bmGJCO5{VNUkFdNtFyNSHtPpA?KCSS)ZD6rYvd>x%vMdO9 z)Oc{^I=nkj`BJsemMkTUIKgG0FsC^PC%_d>pThOS4OMbb3aJnCF(x*5>V>!a2Z;6S z0axqm1gRBz_DuHBnWe0V_>!4vrrO=o>eZC;z2^k?NK@0da9YhaJ9^E?Q|{nVu2tvF z^|7EV5E6{)Q<2z_;2z|nMN9mQD?!RALL44QeDi#no&dRf)<9L#{zLD?fJMuW(||LRj|8UxJpP`x9?!t|{Qz4B%Xw96Y;CoDd(WIE z>@ME5VR=sJ?2sZsRH~TpO6eNxfF11@_;O>ANDH|R@LF6$um zJBGJ2#N8>n{%)|v&n#uz3-@veRbhBX>Mz>_+&>YP&K^Fs?4~|fBRX3-FRt-#QbSdp zfDFmNMhKnm^(dM3M^2@3i;LOkNt3W9Qoo+8FoBf^VE7ZpHmO>>tlJZGjh~f=4_nUw43YO;Io;w=huf=DT*JTRPCBK%%bBdFHwFi_5LkC(aX3e zmX|rTDm!sWQbuNiZS9xS!;yn?$Lr1vEzs#e3;b*zPbh1oa1}T?@`OnCCa4(L4?{X% zra&SW$=ID(KxG$%bP-xaHj0590jW|enh>Bc_lLD+Y?5MY*|t)kxzF$|g4M^6%SWT7 z(O5vgyfVjvpeizEHbn`T*=7TyHkxV4#CqV|qQ}CuT`e2GSxrWGMw(!i*3QbkdVWgB zF(kROK8HZXtf+hF>T!LGQ;ZFTx9%EOLz%Am{=-cJ^n~_k5z6aoIFv zZ}rjVo^EJ+ZUJnyv(#~tGjeOpZlx~tW9Vx54dhh@EB!_pQQC46wFW07E^#X6WYuBi zgG~8E&MjlXk1FulS2n{@N{7U+vl<|vQ9KlApcYyF;^3A;OEU-+C2z{jU0#LN>{8w3n|T8dACY%Jz9*ejY!{#>GA8%vS+{mz%2z!VK57rCi@R{;<|<{*rFUT?M~))b$>yQ z47^#)=}PF0>lMk?SMLEBqpPc+z`{7l-&nY2Y;~F=;N@Kf1`@g|_s}rS@YS4xlOdy` z1&-+0QP3~|5|*?APX*Ld9r-D&h7h#h$ zzr@<;Y9{tjx-zKNn%>q!>MfA9PR|9r9sBp5S_d7$<0#g1{m`h|WLo25Nwlcil{5 z^Xj}c{9LI)l~ZY7QI&31Qv7&68%wQ&F7pMJe%7_i|H9s{Kr&uZr+B?3G{gFRI6_wG zHeL^s0ZUWKLiXy5fEs0Fj*(Dbt>x9P1UE|&u5BPyDiRlt?UKXgaBAH}@l|xPoF5g% zPX<*5Z5lhnFp8dnu~3$T>QX!3DzC_C8gwsSG)kkMgSd~eJh(C`os3Br`HQI{*7%b; z8Z6r}o?TGzM#!MeV)phB4zkRzNCCm4j<5>R*ulzg%az+^j{Z0dj@!0qzf2B8?zi}N zZDZC4irwZS;hB)&d-vg2uUhoJZ(vZIKYB&EO!T=Awa{$; zaItbsQZ7wPtu*+RKwMg=o8!J7mCHR{V`0XHsr)0zw5!H?IKB0O0KMG<#LE;QQqr46 z5`Hbx)~$n8soOx{QyZ3N=6%Z@Dv4sFU!wk2Z$elp_y+PvVYu~KF3b72muMRU2uSIF zUl{&c7V6Z@Tovcgy=9DJ(7wDxze9aO2p$82lufK(A6=EC8ST&D z=t4s6&9ARZ6I$`jwL(U!10{llxs{HjZ$_9k|m;^0RMGBtKsKner-na%zdOvu?fpj{?hjZ@8*TvoBndIBD_kgps zn&zkO$i{%A;Je?^YU9r0px!ARqYk`>oBLHXzl4Z9fzb>Bj#C126&$9Bp^ltw$W zm4xg`9Mp{^YdaZ0&~%{3x!sM=Nf8ufv{Ops38RzbH{fsu^2KB)nDg-mw9Hmf7y~!8 zk!OE^$ra3!r$QG!YUGYTH1t)vZ(OWulCL!=7ck@eG5#7;E9QdY9MgdBN%$A9i1V+$ z>b$(=Fm_f>ykWw3MZ9mFcN85uy!8Ar$#bfdRTVUr&s^$)H}xl(&SR)wL^#h@&q9t1 zWN07!HCzt0M<9dP!%&!A; z6w_-k4s^`=wn#zOq)MTyF3gIgWgxhRQejBV!XN0A_btzGo#fNOag_{evZ9IqZJ=tE z5{t&J@OR(lDivtUX-R=d#5n2FH9UfonV|;C&5);M&NtTj%npmzl&&C&89O%dJ0AfOZ@)M!7c2r|}X zTOW~({j@K0hmr&gP_a8omH>@7838Cq9&RN%`P-he*?EJbW)Wye+olOsxxoeD6# zaBZ=_inip_P6@Zn_}=C%$<3t*jot;b10!i%N*l9tsGZs;dmB-+xN_`lik(WD9q=(gLq_7T|3qia%pkFZXhgW~lUkqukv#$rk8cFr~HXjvU zuG<+ib4sRkNv}91h-Srt_H6ykS|SYz>O=}$H74AJzF_8mBU1RTl@!ayADbCGw;z&K z7Q{`bDTrZ%P9 zZMUK0ve3Q9Y~HcQS!9Yt{ghA#Q^hM^%kexpT|SKMbz3^LFH1{YN}|Y4joF(*$h2D; z6z$aDw56_~l_if~v<>S1iXI8aDIViOd3LmreY#p& z1^Y2egrMF^wrv{m)-Eo96Y})4w{af}d3RVPNUI!gn}l`y&@8A)Qf`q{znO85O`pn& zHHUXLt^-pAHz?bfwhGouAl8mLBb&u}G&A0(;h9oqt|`{#}Ma)X)oq=>tse_ zR)ElADL*uu%5Qm-V00jg8&d1WUi@5##Mo4P;fz+RwkfMV5=r$df!pFapwc`-odgtH zjN47*q$^m+32O>%gEO+ZzYbWaID7{O?GB{do98B{kcNtKueB=ykP-E5G0INao4EKc ziED^44ovC_vYT*Bm=0kQQmEllv#J-f{C@Olz4Bz?kf0RT#8jQ+j1 zJ>pb8*tdv_VVPY)zNb}Dn)=Mxwc(c|u5zm-CDsMBrkB|!TFv#|1VyFcW2)VbYl{W{ zKnRFCZ*{v5+Nb^#S|YPw%wE8brraC(IG@aBHpx_%Xj_F;G$;D8TiLXy5_PUK(h3U3 zwx7(8`j1MQnM8I-OMQ1=I?1$Y2w%lasANF)-Pq3*vv9sjt|d`bL&hS9ExSxAkMa>G z0@)G>BsQKJ4R)hn2i37hvuGurU^U?xLQXh;6=AW>imo}wfoRBsMjtq0?i zgjTYou)`teSN95~w(bC49ISahw7a^FXp6<`eZ+Q>~6#y4;5LK{y_ z8olP1PE$?k6rUwxjjA}c$sGboN_Pb*8T>$kEiAU~bXr;%GeLCN{DcIR$Xdv{qYwuw zol~_1alO4m?yK}V8(6K7>>SR(YPp1pmvn}Mw5>eaYn2?dBKMsmWSV<+o5pfrZ85|H zuDfm3DXV-#|F{Y}PIYZT0oOckQt=*R($t~@RZ3N+oU-@@j44h7+N9WL%R{wH*WhQU zR6V*Y`SnB3B1+I;pm;Z1G3pHDNI3h$t97f*+fSlpL^0^aC`-;@NY}wcQ}b6*{zyx* z_zOa!v|(Q-Pv1UgZ{i5wrvv)7_SvO?lzfm@=$dCMbzKYR=u~b=32t9ctZFvoJ{gj) zGD#iW#JJooPcGjUrnx z7aVt%t&vbijRY{Y2V__(cp_o}fEF9=;lQ2ve7G7qOcl~oZE0FioP$%naon9@_~Uz` ziU9?(KDpxuCTh2MWskr{E{O|n>ywKD6rn{3Tu021NFoGmNFu{ZNUPbFzDQt^Eh(ti z_pKB~!(~09vY$+PmsDn^E*%*)mg-vM_haJM!{V3X`GY}fzS~_O^XJn^as*bZvd#!K z+TU+n(LcKwr(h)vC$oZ9Q%NmyMUKqfVs8Q_9tglE2w_dkJo;g1=$N&|^`ixlfNL9i za2|&Cr1dw2qv_*fE}z36qAT%UZ6q=A?1><|+x;cRBU*OFMmd30im9CnScTQj_*jMw z8+hCcmzVrKE-6TEq_{vU6)ITdIi0NK>zl-6A|!BtM=98gCEw`(BkK^#zW z)MY37XC}h}G*-{2t*Iz>Y+#39B2Z8drRr^_Zc!oERwOA=Yo}~z-3>$v7y6j8cpcffm|?KIb~xW>uCTCvO<_DFB$vyPFw7?SLTB7x52Q2` zUq}|@Lbc*v`-mhw0SN83jTK;}TSvk?S_ZyCbp&_@-xb!F;oeoesUri8= zs;}YJme2|k#;Y`X;62ibe2MspT2w7+_f@AxO9Q!8rzUKj3{-l8-9_KvV^|&QPZGRCBn@F&!r=l*Tq?a_*_!C}#jejeQ@D`V84- zov3wDz`N2H&U@&urxB^dLp;gN%0vk#+DNRRt2A#v>S^FTejECnF2%K`wlI07wpNqe z;o@xWwALtmV#$zC3pceIXp(FeHsH}OHE>I1KvG;#*Z+FiUi$1ST~mnF<7|*E_&l`Xouyq2|`1z|%{<;sRZUdL)_o zd!TjOj)8eU)STGqbEI!@r>?3%BZtd~7hJEy(*gZSU zV*pF*3(w1{jX1w^B%Sn8?grcSCV1LFO7%8y7?xU;#OVY&;%jaxu1v%F35WdXIK&wYoAn>NBs<9RD(1f*)L~xIw~2oMA7Fb zu`R6aI1)Q$l2arW(nf->FQEC3htLi9hcl$U;qd7W!aa>QUEK_m`Ix<;_)NV^xN<5is zpj8uvJfMr7(86xQ1!Ar)D8}`x|J_El)93P5?+rWuy*_FgR<{^t)BCXi*Jrtk3Q&vv zZG($#P?D@CA1r1DH~t@-)9+I|+dpyC+$=Z5LXz>rI*;9m^A)7CBP=JyQ|cv_g94Ij zW-zAVFy?jJ0$~PC2e(0b76ppbF{Ha^ql5Chg!y{NEtR&+FMlfa#9I~^G5VT8k)O(e#*}^;B$g1%9rpj>CTmMgbF8 z$oRS!Ffg;a^ZZF)v4lf_aIUK-FrmKCMrFYE*!rZ2HzXUiHC#It-N98G5G*-2-4a|Q zci}Y~{dCYfS=$2*8gqWbNHI=dmL2XQm?%-(I|9+2=n3?73iNt6Mqn3uohkz?l3}*V zJF^HMOlcgR5Is!1(FOFH&@B+`vf5vvbl5HOXNB*_~M405m4_eyKfSf4uB8 zy0K|((6!pekduv$)wawfIat6DjvTA5a4q&Y*(y~?jcLBo3OaqWK!a`QDO2wlvL}I< znKMa11yp4BjC{DIKhbqA7mEq<$p04jDL+5KGdG%)O#IhZRRloGxkW??s|<^ux-pQz zKrXP$0`GnRUt(?j_3*Lwi+)Bx>hU9%oHkckFl(u zm_BZ$*y*f@!O}G`BdXsN%N7+2DHG)iARCLGommWRCb$F0L3tI$BL>y}oUg8TI7^as zK_0%Wmr%VXS9Jb_PjJ%lJ@tnOj1K-ITVM#(l#7{qb^MM_b`V7Qxds*VCPnw!F#8_+ z>?SBvnF5Ugo7A~7dj)E~5PPuPE);pqD2z|kApFxVl(HrfV?05xlF4qfW z%hE*fPn|=<7fQ=}4N!ML`5`aq7y%G6X}Vt|jXnQoE3cKKd0f1tT~kj|zICCAx?I2A zqJ!XNDMzHPjw@UTK!ZK6b&3xMd`gHelQOy0Zgm#(k z(?es>*kX7+B0m@_b@!;F@zJsGIljN{glqJ(Y#6r&@(#H&*iQfx zdJuj#G{lvg9;A6rV3XUJ_eL+JQ64Lf4ScS~Ms_1A%x_Nre_FfBs4TawOGt-wcO%{1 z-QC?G(j@}Yosv@00@B^x-3<~F5&}x+_i_@ho^w3+yD%7o_Xlguz4pW2d+legHK#(~ z2)*bl_k=}x4M7?9bxw1y2U*xj>2PNJnL5Npb?)(uAKO?#FFJ#aIbGzkKw96b2gb=) zk8n^qsN4Ujw{epNz8(YY3fND9fVlqKL)lMU)v0MIp@^Z}CV`n^_T*vvyg^_hdtRQ- z2+Wd{?l(|gYh2e$11*}8`Xat<>8%qjWL@Ib2I;{|DP6_5eq>A3T*0_Lp|r1{U(E_7 zH;(tpAwlZF@pG?^_ftw*94@8LKHlAW;Qz}CvEn^7Cbq{ zq_EkX_2|)EQ?N~SM;QDRJ8obt`<=^^bVtKEiZPveZM2HxD=IW5`YkEB7TV-r4OdE+ z#He;Akz*FT+K0_wPGF?r{j@c)h&cml6dn6wyG~+>L8M_)V}I!Flgq&l)6L_s7wDB7 zf>}$w7M`Z6188=Z*hkedPNV}(&0suRyJ`FMD25XY`Sx`&tNTec$}71n&!4Z8kVtHs z7caWb5fMhg)Mstg_2ABnzuMTuJdJo+Z|5OT7b_lpg$Ad4D7~iHP8~jhCswz`6|YWW z5V6`CWV9(0iE_4sH!?qr)Nxj}wBTmppPIH6V5Vv9{cK^ZUp3Iz`_nxCF(Nx|dyR2G z#y%wm)H^2ooV+(ZRv#H$+ffFsLe@{GUwR0TuJghGOA9WVxFr7%A59%1gWed8PM8tyh>4seW!W15|S1TDoEpVA}s5l z>7g8dlm0+&AepFUi~AC)+6Tm%9tY*voFjF3Cvj^udjDuhJ79uiEP!TQP5oQavlMyN z{WroiLO?5+Ec>qtKp-j+pYnF#8PpSuhtoF#>9xsFGp2Y5@+BCV46Q=PA7=PO9x`PshFY|R^h}vAjm$KSb3dOUYq$U~(&yBcC zCKTBE-wU;~!6u}spc$D5kt!w2DFmL6SfHA+aI(dA&!7g*ba|`iymVzcw4aVUq!BW7 ziDj4!4TzhC6}gdS zn-k5nG(~|@hmhm*WVd+M=j$EFJd`1jl*=>8ID4{VCN@iOZ9K8vK@ph=5ntTBrMKG= z7*wRF@`h|J2hv^lSSs7oKUxpKZr!P+OrW*|>p}o^5JoFV6p+2O22!q;m&!^~(3!A* zy2lEo!Y*p^VstKGUHMrT6ub8{e{=ww1saVh4|7S#ME%cU5Oy2zBt8^-8S9_2eb z>d9uGB(A(0%ppUPO6bpDdHjf-sG@w^cu0g%o~f3q71sK0<=x5cX|>&=;+WsWDH}a2 zh1$z0F`Hm39b(heWUC5+eCzJ{67@v6h2IHu3L=4Rg`*M|ZyW)7*1QYv*~qx{$6T|q zmqeAw???TgWSj%voI?qngAuEDv){HGw`vMSzeRd3Xeq?C|B0lJ(Q2Z2?Ai z7trBy)V~k3=jNJ9(y`?mwcFXCeDHpm*3zk=^La@qyKc_1bCQQ}-aLM8jRm0QIp#!J zEZ;i)lA<$|IkZA2{Pd&b90i@YT0BP3jzuiUtgEae)BLHB|KKS@^ z^jSq2ITQf|k1BlFhIB}ukRT%3C)}gL>muQO(3y#FWTo~i(_9N&3&y#xK7XF`!(WX& zGG1%LUx{R#4#NYgfIxi8GWD&X9?&-9@^*cf4M?s`Q4A9X_zPIATwa0kiX4=qFvWWf z!C5N}bSRD}1HItpJxwkI4<2QrN(sa8!}p-1%M7JOf~mddU~BBQr5hlxKuJ&6&8kvngG>;)FYlqQ6tc+;BX%Yp8RGA~j=&jA;lk z(khcyibKjB=}A@)h7@D*L<%ijKprdES>qmv=@y^v#wRJP^3-QSFThPb<9(Ge-Cc$w z+1~DH+9*3&jJcy+(VxpOEmcxvkTbxIRlK3BOzeH+*!(UuR(2;9_6Ad;q(WDYLpL(I3<$10kP1!A^|h0npsVMF zp`Tg8cmQc^RLkq&A|K0UC=8)iJ;U`leZ8$1{ddGuJ$n4_LSc|aEa5%Z_WRiK#9LVg zOX=24qgg^ZPLXjMI#^%(HPp7XGFIrAol=@o99bmQSsNhMs6BUgLVWtH>X)98f8HX^ zg&GQm0}d=sNPk-d0^q9O^UBSB<9|QqWB&y|`zNlJrT~d7vI@%GlJjRk=YZ8QkgpV$ zoxJ8XCoBu zyJo|2;&xFIl$x-79+K3voQ=#1f^ zq%QII(8HxSv88@Sx!svfW!VXl^UKPz^ya>3WmgW3>L?-su8Yo$oOFlEs@odYvw%!C zX+p3iX@ZBi$V^i3xU0RMD9-OJ3Yc7az)_CTeO_Cp?Lz_!{g4Bm%ZvABwxuR10-tN( z1Y0<)HHlU?-up52d%U&^U!Y9G-in~S`X()M8BM4l0t?7<(vs$61CZ~YgvZ(idBaI& zh8ie|)6_HWAw`5x_WL2xedhGpk_m7Nt^0M{mJR&p?^t#0srUD~>;DBYI2Othd8FUtZZ>!&SO zG9jt=ID@)#sfDJ_)F?D;)6CA;d0k`Ze&US67h=Yu&g#(U=(nWy*R9RgN zvOc%U#1ft%R851No8uN30WgHSH^_nSX+vYBE^ThqJjAQPVyd^Yg$H9zHiAKg)B;lk zwKl!t`sYN-O~q?97Dn~?&1b!ruuchz2z>`&k$070ShNLXh4A9RvBBWf?D$0#d;>wT z0I4i-vc_P(t5`>aYb4<{i8|cOA+K+HJZ-}Egm;r$X$Yw1F5#d#-KVrk-@75kaxyY^azs`(eqfoq2QAdLn zo2gV(S=5%|&?JPxm{&usM-{qJ+bMg8r-l7_EYZKio{G>C^yW;T4&xNjDHtz|lzejj z6i$7&tqHE*W71tD1&dihOKiP0KpfsSn;B_jjidE_w(zFX2(cg&$q5L2`t*BU63Gm@ zILEap)@jC)ebEJ-NnS5T$3t4yMW23aVsYc88hK>L54X>s@b(}m2wd7QlStJagSiEb zTDElLVsZ25Y0c$u`(f3GU+VPpSmpLDNnQ!*Y1XD<@Ujt`3_A$%5pYu;EcV2683(Kh zEkeXeOJ$~o8?s?{FtB=KaEWbgO3dyFV!FZ7n2SLGa#<7dMXv~1dqJKT!GX4kDW%vN zSRVzVBF8<3T%vGJkQ91?A_xw#tMgqsa@Woe*ac~BGOILGXj?#bFj1GbkM^_Pt2eGe ztdpFW`%-ftvE`GeE>=!PKVf91Onr&3h3^a7zg>#r&Fs#9B{eZXpVUw9JlNrC4J*Ak980PA z6XG2`2#T<;%d#o$O!t|r%lNZdu#3&SV~frObA?Y*8KRX#kza^LjE@ipIWNBSdVhHG zpLCQ-4`arU5I{g9s6arn|D!X={r%=oB)7lSP0%7?k$@A^FB?MJOXCpwnIHqouxc2p z+X712yAln_UqDzOtczp4WCmF;{`{tCXuLAKZw;_kno>IR&t_CPEII%RC%kW&czBv@ zOc4yDUxf8@B%J{YCu$C858&@E4-(jbxqY;FnT-ZR37C)A$yY7#7cr0rpr9o1NwPa5 zXbEY$zv*K1iqm`KA@%t5y1itj#Y0{@@$r+Ob6(B#!@n$*NV!1t-0P4#R64q>faYHB z7&(GD`Z^$a4Lt%~u;uQGM+a#^cX^1Uix#OTbsqk-WJLwvd5s0C1ct07?l8dbBcf&G z2g<3@Oy8b)E{O6zJRVf?UL8tZ?j(6eOctR-4(xrq<(Nuk&gZ8th_c7qqTHF;aodSl zm`aBAX%S%->y{*0SPWHw^dUtlD>uGB`{ihOuY{14B^my94Qo9bD?Ac+KvEx>bo;Q1 ztEHQ;I6fn@5~f7n+PR-$_9TY)Faj$p6bqcvgwk6hO;Max+z~>WoVOpzYJG;@Lw(6| zlO|O@GpDSgOU6+MS6U-j_?RUY*J;r;+SyRC&Dz6U-O!Ney@ecYWyOqClMzboUs_~A znq63-wN1Qw#=TqRScj|4broSp8m)p+LDsguGRNYk&v7`kn{5U`0=2(e?JGoGgdwTO zqT*}E*Qnb7sREjdi=kQg@uVphQ=yZP^|nT2+?*KrviueYEnRdvB=^wpzv(w9H{ey97-a z=R4(Ea7w6l@q9ZyN=<@XEIU<_hv)2VwDoei`RXl2> zZ@=LBoMajwUCDt}M#&|@!JZp|R-$mGkN<&;;XTPWLT-EfcFe7?zFg$fMVJ#rp5ze9 z?rfPIMJ0Dc6`T;Vln=dZ>1N@1^ZCIc$~DwQE->0AdmWybug1t6)W$?->w}!H+G%DC zFzN}{2@*FV8gvtUHb^lM_Xr*1H{0XKLY9znjT`-Ipg`{e8Ohe|j(c6jfR1Qs2hM2& zpu$}wlraxhJ!rp#_JB+K8XbzON_7g`-SKw^o{Ix|(7A_EG2Z&|Ilw`RpNEFY9p;CP z*ow4oG2Vt)V9FVax2s^z9Quc0YQLeUA`axhoMGtM7wFqtrdkaMqhp{;IbmzRy;`GQ zKd9|{p^-mHgj0JrNz7e6T5n>WnWVyDLhoO23BUwb1i{Dy`d?5F^3@%4V7gTQLlp+dCOU4)Fj%s8BH`q?!3g6$c6UtN26_Cd) z(uyFtaSwBJiij|y-)?NYqrypewX5Y_(x<;_VnV|xb5c%fJ(6h5FO`dIiLa(f8z8m4 zmyPPt{6`o9{uQZz%%=4DisWK>ZoKU<+}GH+ydia)`E6qXAQp5hEZo)zN(Njp^)Wn!%fedjrPwm zOKs+=xt!-jzGyt1a0#NAYhqPQF6Xh(#1Fm}xEl93R_x+EpPs04bPA4|#Zh8P%IN-j z#%pA3$`oI2R_fbwnhad4KxS=}78lI5j8*3Wbgd-<(RRQawf*e;!pZYh?3qMiz$QQ8 z40V68CiBNq*?8dCfaK*B2QRC4ooseqVo10U?+~9_IMY_1`3fd$lFc0AoY3XL$wPtf8#y%RS^rE45;33W_C$E2Gb))7rrt+~hch zgjWj2-rlvxfK!b!AL)!^AQGYJo(JDUjRT2$y(^rMwZ6_>&XboWhPqpoK|c9L$Z5cF zc@a(!geMZ(4Orzklhmm~W=*`vt25o1X+aQi@hv;iFg-*@PRj2Od>IODA0lBK*lLF z-agDwx)t;*!9t}umdPO_Wc)dtegi3ZEr(gvD4#07E%FcTX`H_XXewdhjtOtzt+b7bOI1>Iib*~)Rc{>nb zfIwGYXA0C7l{mQucpEo9E6GwYIH zu@UwXMNn}-BO~0!_WY)B0vzuR1U3nGm2lR<>XmFhJ*FhE3Xf4sRs#Oh(;=fRTMCrn zjQqIQ4zA{7q}9AN>iDvb_2F2T5L?Il+|q;OC2~BTKwaZzFro-1@VvvZ7$e$+aY=Nl zSJXh>N%J}!+}vP&9$?lp=(aRuiMUV}1L^j$eSyJjqDA}cn$&X}+Rt2$C~j-8gcC{k z{BB%=q_%6|Z944m+>euOw-x?_x zhAb6J5Jk~7?$&d#-}thk;r4R>8#>TgnT80|j5c+Hx?P!Q;+vyZJfv-}m28bcg&29y z!pd?vjQQC`^4Mx=F(gyaoQ)6E)&tq3^N7UXyrD!Y>5$@HqwDvbs_d+RPmJWm2ScA# z?{YP0;9x8fyTT2vsFI8;=lWR+Ouic22P6A@VMx&4e0-H@G}wCZA>;U?sOR+O%ciol zyNWMO^CeD}e8%{Qxh9q>)P*=slf563sG&9@`r+uLdzUT}4w> ztg7lJ%r!N`S8Z#-d*M;jt6p(FEoXOD6q(ZMF_UDIv(6?~$x>YIZl5xIo~@qZTdJd8 z$T+vq6uR~Wq9Hopb;Q@#N|@hBmF!QF5mk!0c_ukEFcMPV#ygY=xY{Z1BP5Z#sQU%> zpc8Ymt178`ceF6shV-EH_u`{ZfVC%p`C&%5r%~!T>>_Wk2c4el3=6mgPT;&+2V?hj zxKQZ;8Ak7O2{ASszM_x5ycI7{%I2)4*K|Tma7{C?4(^Eq)$Y;&pPoIWwoEj71uyo^ z$_4R5ZKJM%De8<=+`;{$_iLFx^?%4jKmi^-84+ax8VOm^U#r_=e?|iW*=OkMI6!@+ z2=D}>JTxIAAS)p%qNGeKBl?G^!%Y2m&B*uD|^k8&H;n`48wV*zNeo+eL1L+3Kc_$}tr=BWhI7 zfZ@2$=IEu$=$rJ2TVQ+Y$6r)8Vt*7IBWEq$acVA%&!>GCkC>*2euEifxctcruKR1x z1=Z&84YnnMyn_wFQIEgk+X*zCv5ws@R@WueT~L4ici+qFR93D9ya2ok)1?L!JUxt$ zw1kj|tdhtdQaeA0&61{;!$LQz=UK_DeXf!Pcy21Q1@=~O*{6UaAq(L;YqFR?(o%vN z@X}sdf{>dNo`fdvY-5*8Rhe8CiM_dst*@yEDP1r}buA)v1F1*{PWhbgb-BNN;)`{f zcU;@O*cH5%an7oQI6B##`N&;o&IWsvtX6V*b)!Ab>R?n;g0OR| z-ukxvIny`tB&eDg8DdRRE{0YK2oo)GW3}LMN7NFesxm{z0tD5QX=S5eJ#<|ZUM&kY z^(M@tSA?l~wo)p=Ljr{=eS*y(7t>)2spqnZSiI)0!?8vm>Q~k2jO0+LWL}rLHDLSA zx%5NJL!s@g5`1Fc3!A#jy=hEwpbALEd16&xg@op;<<`4EsKo!mgt%DZI2g$&5Q1bs zE{aqFO4AE-MT4%p6;xNC2*(Bv9JkN{ny?5OA4{>mbz-H)NUg1Wyt3$XBgtL*?5;tN zy=zCEs~Pp@vB+vR1cttXvDG#)COQ*QF}pQEy|0QDYSlnz8iz>FGPrJC2*HUZp=5iu z8c}Y8sr5yuxaw54xff@`H-@QHgKlKUj@qs&OGxJGOir<~pt7~iNBtJpo$JLUs*2l^ zn#xH-n$J&94(KenKsOCKAOzM|z)LwVffkM9V2}hpi8|@~Df&{8k2CSa47_Gtm)368 zyiQ2f+0+fZ8zp;MkEIZg`qJGMcn7g^3LX`>NoyYmjJpey9Jp?SWOBMx3yKo*^Aw(# zrj1Hbq5sD5a~ZTdKds8#aa(2y4OE=MhH}h4lqkPUCSP;M4|XnxSxhITJ%{#{gzPyW zyMc?u0=lq#@_yGPC(f0}T(vn0nC^>Kw_q8z1@(LI*SNJtRup`e&o$4>7W?2U;h{hj z#yH#+1H3YAu#onsac%m&NF-?xwmv#n2kkZfBWttkmQQYE7y)wvO`8sky z#!x^@V|inZJN><^^}$`)+p)Ev@urPcIbhYbdIbfv=+*$&DtIOf9mAyv>A3AF=xZ^d z&1$9Jh)I-;P80f|)%tfJO4odm%n{hQB7hdx8{w`$H<45e$h#Bca+YK%7ur4z^9;i4 zezq90=wDa*@w0t`H!%BiFD#(97E``tYKADj7Vc?~U}&tY?jVTELoZoh8mMo?`ic|o zO3e;3$D))<+c-fqq+DL3M1smr#T?UhmR&TS&?hw=h&PsCsa7fWTq`$KqEMQnbBcZ_ zw1kg>SR!X!1kVIp&yFdohn1ZH60VLXMGYqU5IvShE}7vJn;$dtktp^G>-F?{4+nx0 z>jE9tD7kiCj>1zE)+4Ac+l{y4QO2{ZmC&BAB`=7iLVy)Mwoc1>5vOP~PQtw1!&-X!XrDyY`sB^ux8)cUUNx-EDWa57eFVLg;shqz z0Y_)!rxegsEhj>?hhWQ_-avq7aZy$mlq`t8g>vRo>e!d|}$B6M(Sitl}CARIN zy`%DmMNjh!9=Jjl24ZIKo7KA8K<5m>W|vPdJ}p3L||u1jeXN# zIsn#zFyOjK5843p&x>Sr3g>gf)5;6o(BEuR zy2Omjo>~|-yP-mbo!_>6ZRN~fN(9wcgl$*hyXPdUDKn>`#s6+}i~`qEFYO%t3(~y) z1f?D#@U2(0u&UfZ1iBxnUEuBqn!FJbn}U>&hLOdPRo1leESG$zg78t$Mx2}`qgtv0 z)}*)61Y6i=p~|I1HPzj93C8r?q86f`XlzD#yADC))@_^P+7|7VUy|#6;Kj&L|2FDb|n`eEL?or1$OI1kjz$Csj0HVF&zSG zRp^&W(a})QV}v=$eAtl$!`lw9`R56V*jZ*#0{tohkTlAe++;~HOvCW`wnq}OQYVw< zQ=~qs$1OJ3qf-@ALpNXdPhj61*b%%kn!n(QvNyZ>G#4k(`c|5wjQWBK8C6pbiZ>SZ zB$0A^Qlo;1LL%jxsUdeBBIBXJHe^@AEDN&3a=Y2SWE(gm7AbDE(nahtox~h;CnT!} z@?@+)kMXW?hJ5ifTx0B(w97gGh%jA)gMz3&P41)CDn+{*geDc=%%>Kz2kBFtD zS3Y;=e37V=Q`+=fnN@E=`)jQ|m#;yC*-xgL{F_OTp{tug#Xgy3dP4VLtK`&MT7kYu z6_y4<`*P>Q70D0|ixYsMA5hUJJn?M%Z6d_%5&P{dz1)&k3X8(dv!Lqn(ln^nu@x!@ zFzBPLUX||X6wzr;gE}(|cdY%?Iz~f=Ii2YI5p$*N%KUr{Ztk&=PZM>Ck|a-ShI3y^ zc*iUhNT5a(Vw?2|=P*Q&8NHp&$8iEd4s@3qifDma(Pb&)XG6#5%*%{O$;C6sQZ%}f zLaKmy!~T&Sr=r&?F)q9;4M&GGHy$%zM2a=;ViX}8;__s6fO^+Qh=iO!~n~f_6VDyVFj2{ouOJ-Yx^_B;Zt6r zJYuEDfov;#L$8BTb|%*R|<_2#ay>A?-h0##4G+gg+4vP?Z3uSUc7e(j?J&j6KKkPOOJ zI{DHlG)~{EQG7@$$p^L1fcH#C@dU)FE2GUWG>I>KeIVF@&aN0NichPs%bo|Bk9vB* ziT+E^#!Vy}{+F(vuB9bIr?f}Wqz74@5dIepkKX|oqbF~Xn*z8uP-PE8}eCu{IakA5lz^C)E=?1Y< z`?}PRM`$NZZjDhmT?D8KpD7yr2I8$=g?*j_X|gOc0@e3)+%Sez2R8S>OR<=+qAgc zvI~)23#Ngxvjh%zc(N{$gQ~^t0NRlR^_O6y@2m-U)WO}SMoW<${baeYtNfCwA=3%* zF&!-z%=Pth>Z#XWxG-RDboY&zt>5?rgA5;IHxUYo^}n#gLp-2qj|Xk`UfFiOcK4)r zlkJCqH}M(3lEJ!+xaAM- z*ZmfIJAB=8z-3dt%c!VZYkgfbp^!r~4D$ke>7|)zZIT=;ZCVrZQ_a;f><-`tmfZPW zvUj+iKABdRx@)lr9Bn}%Gm30k8I?Chb>2+hp1}12+4vpW`=XCv9xiE9OBO+oZadKXM zIJaBfDmakJzzvoovGu7u)wH;E!BWH(L|zQ5zgXElD6JJ2l3Q3b1j ziver+J3&O7m;R-`g9#;?E8K~6%`M}=0!F?^S#w6@okhmX7;{X>DZ^qmm5wsb*hm~T zEod2p#dlcV%xx5(rKHK1GD(ExVlt^=uca|wk?An;8nrt#$Gq4vPnMh-Ek1S+jlsiWI}2YL(iTngih2QZLai*a*hJiL|I|=UGs)^hp~$4 zuzQuLsi~`tG=Yn?&{^B8MsM(aOoy@11PXFHETQ{U@@3T~yPtQNwC&>!IxWTZbCd}S z!SpHc@@_$FUcY?NXwD1G54rXr&`(IXd~d;RC{cws_Y(BX5YNpSS3fKfK0(eXIJhjA zb)%&%mOcwSZ|0BB5;#Ag1#{Y+%;ii+M}Xo#%ED{r>g{)SW(%!bgZJ8sJR*usa#IDe z1p&M|q02kB;3J2lkR7pKGiUOCxp-u6&XRZ~6O*1G; zTz0=q(36*eYdX6)aO&D0Y0dG^tfiZ5fMh9_zrE)Fk=9?x~Q`j=T&MRI} zXqFTp7u|yJjP}s`e0XEyjDqNx81jjB^YuCRsO!=NYiv4$sB$(tk<*Q-&;%m~=A1u& zyb$EsNR#~~UhS2o#|@HlE~#Go=({z%+f*i$nD9y2kGAbnzSrLGG0yMATR@MSj z+Qzj?Apt{2FX@S@o3&=ksb?GESF?9b9>ZPLAn2M9@1qp|gkG#TG1PApmE)oatU&!W z^QdgHG4wXdZOb8o%D1WcIv_JIt>-2xGNBg_S`|0Nb+^zpZ0Dl?V?;ihPcm1b2?LQCwdpXMAf!xnz7oZd4 zKLWY$UJ7t@FHZTx5&!x;elJb=&=C=!kH$|p`hCE^IQ)TD@C zzYF@Gz#hwRJp@LM{BuG7BNUH;{Z7jHIT(*6njXS<10W0i%-sJ4jK30W`Z-vSwQe55 zS^<3cf5Q3;<(r>F_E;+9Au=wU|2ML~5Dxh{WRLX~9wJM}|9>O{vohy zqW?Fr`xnFCDXsh+t{0`h0k`Ok&yaj&w6;6mB|=ivTb+#a{ZcnGaQ z@V|xjMtY^)1y=drfc;Iy;9ovlk7?NtabaowZ*V;%Y5!cT9&@-JV)8cn-(mUz zqwDYB{kmm;&-3{m-u;$(Z36U=^#d?>&-(eB9qm5Pe$4s#)f|BS1K82{Odptg!~gwXqUpJ!f*)k6;>zO^U{0DdDe)xZn_xsDdhgQ*C9$5YTYVWU%f`J3Z2t)z+)&t}k KGk`w@0{TCAtviPR literal 0 HcmV?d00001 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala new file mode 100644 index 0000000000000..ee8e6f4f78be5 --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveUDFDynamicLoadSuite.scala @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive + +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression} +import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper +import org.apache.spark.sql.hive.test.TestHiveSingleton +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.sql.types.{IntegerType, StringType} +import org.apache.spark.util.Utils + +class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveSingleton { + + case class UDFTestInformation( + identifier: String, + funcName: String, + className: String, + fnVerifyQuery: () => Unit, + fnCreateHiveUDFExpression: () => Expression) + + private val udfTestInfos: Seq[UDFTestInformation] = Array( + // UDF + // UDFExampleAdd2 is slightly modified version of UDFExampleAdd in hive/contrib, + // which adds two integers or doubles. + UDFTestInformation( + "UDF", + "udf_add2", + "org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2", + () => { + checkAnswer(sql("SELECT udf_add2(1, 2)"), Row(3) :: Nil) + }, + () => { + HiveSimpleUDF( + "default.udf_add2", + HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.UDFExampleAdd2"), + Array( + AttributeReference("a", IntegerType, nullable = false)(), + AttributeReference("b", IntegerType, nullable = false)())) + }), + + // GenericUDF + // GenericUDFTrim2 is cloned version of GenericUDFTrim in hive/contrib. + UDFTestInformation( + "GENERIC_UDF", + "generic_udf_trim2", + "org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2", + () => { + checkAnswer(sql("SELECT generic_udf_trim2(' hello ')"), Row("hello") :: Nil) + }, + () => { + HiveGenericUDF( + "default.generic_udf_trim2", + HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udf.example.GenericUDFTrim2"), + Array(AttributeReference("a", StringType, nullable = false)()) + ) + } + ), + + // AbstractGenericUDAFResolver + // GenericUDAFSum2 is cloned version of GenericUDAFSum in hive/exec. + UDFTestInformation( + "GENERIC_UDAF", + "generic_udaf_sum2", + "org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2", + () => { + import spark.implicits._ + val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3) + .toDF("key", "value").createOrReplaceTempView("t") + checkAnswer(sql("SELECT generic_udaf_sum2(value) FROM t GROUP BY key % 2"), + Row(2) :: Row(4) :: Nil) + }, + () => { + HiveUDAFFunction( + "default.generic_udaf_sum2", + HiveFunctionWrapper("org.apache.hadoop.hive.ql.udf.generic.GenericUDAFSum2"), + Array(AttributeReference("a", IntegerType, nullable = false)()) + ) + } + ), + + // UDAF + // UDAFExampleMax2 is cloned version of UDAFExampleMax in hive/contrib. + UDFTestInformation( + "UDAF", + "udaf_max2", + "org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2", + () => { + import spark.implicits._ + val df = Seq((0: Integer) -> 0, (1: Integer) -> 1, (2: Integer) -> 2, (3: Integer) -> 3) + .toDF("key", "value").createOrReplaceTempView("t") + checkAnswer(sql("SELECT udaf_max2(value) FROM t GROUP BY key % 2"), + Row(2) :: Row(3) :: Nil) + }, + () => { + HiveUDAFFunction( + "default.udaf_max2", + HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax2"), + Array(AttributeReference("a", IntegerType, nullable = false)()), + isUDAFBridgeRequired = true + ) + } + ), + + // GenericUDTF + // GenericUDTFCount3 is slightly modified version of GenericUDTFCount2 in hive/contrib, + // which emits the count for three times. + UDFTestInformation( + "GENERIC_UDTF", + "udtf_count3", + "org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3", + () => { + checkAnswer( + sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), + Row(3) :: Row(3) :: Row(3) :: Nil) + }, + () => { + HiveGenericUDTF( + "default.udtf_count3", + HiveFunctionWrapper("org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3"), + Array.empty[Expression] + ) + } + ) + ) + + udfTestInfos.foreach { udfInfo => + // The test jars are built from below commit: + // https://github.com/HeartSaVioR/hive/commit/12f3f036b6efd0299cd1d457c0c0a65e0fd7e5f2 + // which contain new UDF classes to be dynamically loaded and tested via Spark. + + // This jar file should not be placed to the classpath. + val jarPath = "src/test/noclasspath/hive-test-udfs.jar" + val jarUrl = s"file://${System.getProperty("user.dir")}/$jarPath" + + test("Spark should be able to run Hive UDF using jar regardless of " + + s"current thread context classloader (${udfInfo.identifier}") { + Utils.withContextClassLoader(Utils.getSparkClassLoader) { + withUserDefinedFunction(udfInfo.funcName -> false) { + val sparkClassLoader = Thread.currentThread().getContextClassLoader + + sql(s"CREATE FUNCTION ${udfInfo.funcName} AS '${udfInfo.className}' USING JAR '$jarUrl'") + + assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader) + + // JAR will be loaded at first usage, and it will change the current thread's + // context classloader to jar classloader in sharedState. + // See SessionState.addJar for details. + udfInfo.fnVerifyQuery() + + assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader) + assert(Thread.currentThread().getContextClassLoader eq + spark.sqlContext.sharedState.jarClassLoader) + + val udfExpr = udfInfo.fnCreateHiveUDFExpression() + // force initializing - this is what we do in HiveSessionCatalog + udfExpr.dataType + + // Roll back to the original classloader and run query again. Without this line, the test + // would pass, as thread's context classloader is changed to jar classloader. But thread + // context classloader can be changed from others as well which would fail the query; one + // example is spark-shell, which thread context classloader rolls back automatically. This + // mimics the behavior of spark-shell. + Thread.currentThread().setContextClassLoader(sparkClassLoader) + + udfInfo.fnVerifyQuery() + + val newExpr = udfExpr.makeCopy(udfExpr.productIterator.map(_.asInstanceOf[AnyRef]) + .toArray) + newExpr.dataType + } + } + } + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index aac1ae8366b7c..f69f589447aa7 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -2329,53 +2329,6 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - test("SPARK-26560 Spark should be able to run Hive UDF using jar regardless of " + - "current thread context classloader") { - // force to use Spark classloader as other test (even in other test suites) may change the - // current thread's context classloader to jar classloader - Utils.withContextClassLoader(Utils.getSparkClassLoader) { - withUserDefinedFunction("udtf_count3" -> false) { - val sparkClassLoader = Thread.currentThread().getContextClassLoader - - // This jar file should not be placed to the classpath; GenericUDTFCount3 is slightly - // modified version of GenericUDTFCount2 in hive/contrib, which emits the count for - // three times. - val jarPath = "src/test/noclasspath/TestUDTF-spark-26560.jar" - val jarURL = s"file://${System.getProperty("user.dir")}/$jarPath" - - sql( - s""" - |CREATE FUNCTION udtf_count3 - |AS 'org.apache.hadoop.hive.contrib.udtf.example.GenericUDTFCount3' - |USING JAR '$jarURL' - """.stripMargin) - - assert(Thread.currentThread().getContextClassLoader eq sparkClassLoader) - - // JAR will be loaded at first usage, and it will change the current thread's - // context classloader to jar classloader in sharedState. - // See SessionState.addJar for details. - checkAnswer( - sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Row(3) :: Nil) - - assert(Thread.currentThread().getContextClassLoader ne sparkClassLoader) - assert(Thread.currentThread().getContextClassLoader eq - spark.sqlContext.sharedState.jarClassLoader) - - // Roll back to the original classloader and run query again. Without this line, the test - // would pass, as thread's context classloader is changed to jar classloader. But thread - // context classloader can be changed from others as well which would fail the query; one - // example is spark-shell, which thread context classloader rolls back automatically. This - // mimics the behavior of spark-shell. - Thread.currentThread().setContextClassLoader(sparkClassLoader) - checkAnswer( - sql("SELECT udtf_count3(a) FROM (SELECT 1 AS a FROM src LIMIT 3) t"), - Row(3) :: Row(3) :: Row(3) :: Nil) - } - } - } - test("SPARK-29295: insert overwrite external partition should not have old data") { Seq("true", "false").foreach { convertParquet => withTable("spark29295") { From eb42ff155f220cdf8bb391160d51257bd231b222 Mon Sep 17 00:00:00 2001 From: "Jungtaek Lim (HeartSaVioR)" Date: Wed, 1 Apr 2020 09:19:16 +0900 Subject: [PATCH 2/2] compatibility with scala 2.11 --- sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala index 585c9ed6c7235..33622a1cf82fe 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveShim.scala @@ -21,6 +21,7 @@ import java.io.{InputStream, OutputStream} import java.rmi.server.UID import scala.collection.JavaConverters._ +import scala.language.existentials import scala.language.implicitConversions import scala.reflect.ClassTag