From 8ef84bed4225134c3079859056e44531bad7b129 Mon Sep 17 00:00:00 2001 From: Umut Date: Tue, 2 Jan 2024 17:04:34 +0300 Subject: [PATCH] feat(frontend-python): add relu extension --- .../tutorials/relu/configuration_and_cost.png | Bin 0 -> 39346 bytes docs/howto/configure.md | 4 + docs/tutorial/extensions.md | 131 ++++++++++++++++++ .../concrete-python/concrete/fhe/__init__.py | 1 + .../concrete/fhe/compilation/configuration.py | 8 ++ .../concrete/fhe/extensions/__init__.py | 1 + .../concrete/fhe/extensions/relu.py | 48 +++++++ .../concrete/fhe/mlir/context.py | 69 ++++++++- .../concrete/fhe/mlir/converter.py | 11 +- .../tests/execution/test_relu.py | 106 ++++++++++++++ 10 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 docs/_static/tutorials/relu/configuration_and_cost.png create mode 100644 frontends/concrete-python/concrete/fhe/extensions/relu.py create mode 100644 frontends/concrete-python/tests/execution/test_relu.py diff --git a/docs/_static/tutorials/relu/configuration_and_cost.png b/docs/_static/tutorials/relu/configuration_and_cost.png new file mode 100644 index 0000000000000000000000000000000000000000..d00f93177bf06e59bf6b7b9b2d1d84dab85e525c GIT binary patch literal 39346 zcmeFa2UL}5mNt6Svb02{tU@t@0hAzMAc!PG83;-SC7Mt&NR(`*N>ucKqNrpfgMpk4 zs0fH;$tW2lC&~BOR83FM?Vg_Q`KRy9`tPv1SGnMD&iB3F`|iD;us6Rc9X_yV!O8^; z24j)z!CzDujPK?$7_$TBe~<4petZ2h{v&R)?}&}6#c3P+<5mWYL&t3_O)YFpjZUn! zGqAEYvM}E+C?>dV^IAh28%yh*LPBPLdV!#Y)fpjP-DWqu$q$wXk6JSrtjFp9v%+P< zj2MjjoU*^{Ry+6bYpt8J;?VST-?+(xg42mdzs`H}uS}*hY#Px)i4<6KY#MJnKkpzfKxxx{WBQHlrGPn`Nwj$ zz?uItzTd|)^N;hpm;CQb7>xfr*2R*OYtyE@Q`>lV?o17l-@Wd>!p4m0sqqn)tfx<( zZo99bmFYaXruwS8dtOCF(l?LU3`v{$I21X2J9lbr^va!h`uw@-9X@q74vtuvsc((3 ziAQgFaC_y31nza$(oJ+8wZa?bym?bsHpR)6nVGrltxQ~D!ISk_jT>cOym;aG_37Ja zK?(bAo!@kKy($is_ZgfT?^*R_^*lz+sx+n`w&B8)Z*QvKSSjLt|9*tO+tksbqN1_( zF#mYBY1caf+KC}~{B~tgnib7?KI)GZ?vJO|8gQ{VN)pM>b zEf~;_H$1D~>3B;g)A^{KUFf*Kf_3YMvZ2n3f)XD=T@^Jo?~qly7c&?RytUKra~P7s zRY?|cZ*HuTXX?3*8mLDfQRZ|Ryj>oBG*~ag@s6c!PT{F-cMg6l3YBN~Xve1*6$S0< zC=O+LRlw4=dwxu}{ip93=gwJYHHqs-+&d#2al46v7epXeQeb1bc zhZjr26!SiO_%Ojg9J82HdNUr<{q1Mun9rU)V_Uj3^Z=)ennB*(vJIY+L7V)}?b*6@ z>uC3@s*w$oJsNkLnobR*fBSas%9Se-CvvV?+1RMzkqmzE5Ymda9~W36mgF|rrhEmv zq@dtXY;5ex9hL!tx~Zq0cH_B-IDfOG)x$qE!wx?he9P&^)*C>ZV7M4#hzQF|Ec{R?l!xT(Z}$UE5Z_-pbkyzgC=mVUBu&vAXz<9q+2EwI4~? zC+@)hXgn4|>syn~h)@c==umfI&b-oR2J-3=D*JBVzRjl@`|z%yZX~Y75PP7bD#7Fj z^#@sV<}GfkJmasP;W#90_w@+gvaPKs_?m}DT4QQuW#s}E7S-pc3kGvH`5TrzKIALu z@XWX@GPFX^=@8Rqa?IBD>nG(jyRKrBxQxd;Z9CfZr^kH0U$|VUMwj2p&(9C9y+a>5 zQs-27qLRrif6Bxp%1_EkF7<4m)Y&h;WK8tu8CS(e6ql4N<>ZX3O|y?W^gwh(Cl8OK zG|^03JyJ~;*N$heiiOaK)j1k4Cg0@XPDpsxq9CS_&R)Vr6A@87w-`BKrCB zXRTD52t4#SEUk`qqG{3gLP^!|oig7V=FoW_D%aj=-E#f$p$CsYepKS-;(Aw9v{iFl z?hc>P@YhPmu7rxx1QSi$&hkS{wKzQ~S(TCQTHA)~S&LSQD&Wl0@18hu!sg5CANG4P z&d;Lg-|Z0}FN|k!`2PL-RVmhCK|w)H4GjTj{9VdxPQU(jNu)-Muj^HneP{o&A^7ryo0xbjNHx`Q{+=dsdFa*&<1YVK9yIDC(Bc`dn zomY0OJFTZQaPK^qz8v-(mAAuP)wZ8s&NZt^E^;>U;JU#nlew#N-rTwE^a+a7qpkj9 z{dqzKf!W=)X$dy%D*Jx<#jv9^f@%J+5WA$bF2jjMr*(9o)wZ``_Z=a9J!a{X<9mlX z%H%0_^nQM2ogjog<~Z0ENZSKHC>bb}vCzBg>l@isoz&UrDN`P<9GpLJz>`C_!4dhM z-JFxbs9_Hb42)EZU^+WHmj)kTy~D40*rYsKxhzthyEeIZ^s_r>X_ThW~^WE^f;=G0J}+fm7{F#%XN)2)+DUWy%v6WbLnK0VoO z@ZP&9VE3i`0aH^`;ZyH!rST_Ure|ztXQ%b-)Nc-TQ<$5z*K(l4#eZWnf^i)X~MHz8Y~iVsOw#RxDGjegBV^QO93hU1}^Nl-_$${Ks1% z98!*SB0I}t_-zVq-n{uNDM{1RG+Iz6S*dMk7K4>@&#Z&YMDyB6JjzI`=J%9UG8}CK z%M0`K)y|zuo;_!d!I?8Yoi8-RkSQYZ=2n)LxBUI<=h$*?KJT>Mq(_>({mmU8 z1217M&TuSRD~knHNB#2V`ifv|Iw2vkXDptZ;@bHsUnm$@k~-m zYN*D#(5SsQbnb%r@9zqjWKNGiUa*w!h|ZHWGOjvxb#+!aXojt=t(sgz>>znR8^aMdg8LPcTGot$uf1s)&T z=ru7h(cQ6vv3T*~v9@5(c-Kis2S>+_h%mq65Lqu9?8iM8ER0^uwQT3P+P~y@MCxZ{ zo-XjU)$BZT=1g8uk@72dR!u|0@G{Z2H`kQl@CV#q~*w~bo$7m-Qm24d9sn;JL?o#ck%P2!+(J`|)V`B24r@#2IY2~vKXC3bL zlvY306DNXx`spVwS$QA9vJyoZv$Nwls)|yfNQ4FcGWwoU!}k!%!f@h^>(U)Y&C>$v zALO1Y#wRD7e7ktjnw@HH6MgP_&VBntj7#5r5xMD*+nJV^o14;@%FTie0Q7; z!ZVFR_y{+zO|5!CUG^!tOeCt^_;9ASC8EaS?t{kZndhPuk{xGZ_ zuc&Bwdb&=Em9Laj;`fVIatR3uMOA;B7${T^-oNOOpX5=Tma>n5d&_V@2VCwluD>cT z*Z6tU;ZnEhas2_W793dK_BT;wd-m>)aW!AF$7K^0BA;Km#|Wm0XE_dajG71Uxw1g+ z(4hj6X*~Iqmda0`+GtyO^QsQg$>VhHz2N%o^SsG#jh=I3#f9+unsRb-v74$nr5p$S zl3c_R9wH|iZ&CW#$X9z#Jt!k11G_Qk^5yTiH*V~}0_T-|sIAqdNENit!=}GU7H`pW zQ?V;+)MK)%{!v2DAts00#95c#7xS)pdRB8-s zry>aF2B%NIfAwlFNfo>qrMXMu?}@8 z?Rp$-Dl*y-Hc;gVpZT!bW0daqYvpqT3yZigf7h{_)n}1sj~uy289T>=J+H1#x+3Nb zRZ%(dsIBN^o>MR>i{x&96LVHfH~pM43%g*vq5bEVS9G#E*oQ_o%KEzu-_My?vHdig zv4ppN>U!K?V7xd8{Z;pAWwNS&?VU7A35jN`bG4?bx6ZJ)^AYDW}*?iE&4s z?m-RuKB0EjobP$IZBxm+D^Nm*Qd8eW-L|8&h_?5>0|)rIx%c7YlTW%7N{u`mjhTla zQ-KOy7|&^ReB1;vq{R-!c^Lx2Q2zG9aO`+06{N?%ZdZ@dQdgAhdo9HohI5#$B{J>c52m$W+GNi2fL~g@=)n~{lF$?s z6hx}odtvOB|Na{*DYE;Dn)yym3@8rp|M2tAKXWggaI&_yulQL}G7w3#W3au1UVy!_ znM2Y+;=q9ebHDq}Fqg^QKQiKzmX^lh^mVIln%yBqnaOypQt0{f43wEh*uXno#^TFv zuVkEe=L}?Yx7ANiB|Uro90|r^>onq&W7F*ea^ZXo)=LO&5y&!QNb$mkAJ^jjd8DQF zl$DjQ-Mm?j{LG7Ta4yQ4TKoEf1XsPi>6b2D8p1v>Z2j=aae8uuYMV#WE?T%Xn3N7O`DpT4dL^QQ159aTLz-epgJ-Gb+2*dGlL7=pM|cYL^Q?~q_lXh z5<83?ahc!~tIUKF1q|FE5`P$_(gFWzwJg`k1?m2WAMMOVMsu4S^2yPjXTDO}IhlUv z#wxK6ygpJdBlV3ZXI$3wWY4I1lEVFMxTZT)VBT1?$?8{Wn%Ls1(G9DVi+;+J6W z$nbErg-Ly;3#Av@7E+UgC1YQo7g|Z3>$e+dDJXez{FN~Fs&0m(dQZlPA5z$3X_qlW z{Ag^fGmh>H(7+HiM+f$-vds!c@A<9G-os&^8#1?}0K!!}EN=xkD2>yXS+m1(ZN=U@ z4<9|UUX-^fx#Uo(s_~~Mfk>~X{gRHon2q9;M?fHEN9t?_YXTyD6!tBF zVIZD7rr^A}-nVWA;A@XNh#Ize`xi9aym;}V_oGKK+s}Mx1DZ+i>=L%9ldO=qbMnNA z(sYLbCxIml4#%YW(|7P91h98GIr~ucZ+`Lq7N<$lTfmY-qN1WyiO?=&Ot~vDm?77% zFZ2jo$0=c_@WT&3SlQeAWMyTI4RwaG>z&)}wMkJ!N$J)HKdGpK2VzkG%8o7f&v403 zj~2Sw40WhKl5xYs8uA$xQ5EB?5hy@)bQrt7v$D)Y?&Ev+6GMt5(Dp@^BqwL<+lOQ_uOebfbch5ng9fe%j3FP=b zKi}Km-(SzED?viqhLxLt&ae6t&HgR!r?6WLN7L}iR(`=0P@*K0o zsc>`P{s}KQf5INs0OD{$NM5$t{J9Hbok#nb<`c7Q&z|M|Y)tPVX^KSB+}LJ3uZ7o?X8y~#w0`ENjO>7Oj~U%O5)#_Tw%oek zGcG+(NWdmqGqM4!A=|n=vzqufY@n2rhT2*#m5p)B%gZZ0BLjI?zc>_cCdoE`?yWG8 z0Fz*R@EXMAJKt;<@A@q_H@66T+jjWtQRD;#PwBC6Q`6)*xR3_DB@5On;sf93<|^am zO>J#Qiklf*b|B#1;Z@!L)|0aWzp0*RTFDgr0k6OCws3RuhW+yyJOazt;4_utzf~-q zxe?+r_x=GyZkrEGj?`-+TVAtyKtEM_IDc;MM{%d&XgpC7dU`0#iKjrB@a4;wd^6N5 zrKom(iryyXmFwNpO1};S0pe8=s%Y!n*SH z-Y_JTRL7ynWr8~4z9J@+KC03ko>P9YUL_ zI`8}fvKSjc<#>FC-7!S4;0aJLT)W#&S)=Uh!G{3*I;D;Y>;|XM*+9`x z)U2xQ607t(dnZKs8+9%tx>Wk^Jl7krQQ=+yFoGfKwkwe=ba~fUl32ZeS|w`w{mg~Uy#(v+`rK5Xuk6<9H?|%a z{F!qye1JW!#n@-!^Y*#11s^^fMtoT*YqB zifDzDrV zV-;Hi=bDd~Xs;VltB4_Jhld)DRJ3yY>FdPAQh~96m9^0N(Kdtk_ok}{r%lpaC&zTV zWDs$RZ?yg>Y*L=W_4 zv58lf{`liD*C!3N;ks#wM0t5QBq`RvFDX$4sZ(x|HTf9VRy*X%(pYqW^UjSMH)xZo z$LZBdP9uZRs!V`Lkf-p(;8=}o%17my0QSYKHdUlhe|$jYvBCDexpe7O z#@(aFlc<9`DxZfDI!4q}Bzo1^IUV631RP2Ikexe9k}y9d9Ym%LHV5kFPE^CoUgXz489sfDHmBogv(FH~-+Sz8cd zRBGnru=Zr9{j(PJ=~$pB<7gbqJLmO6+26M>f6Z} zy$mfXM`q8Sy&%3Gj2S9rJ^_IOXT`BYAWC=IeZ6N~8XkyhZJDwnCkHZ4F?5o6L};eq z+${^I^ky^Ixa;5b^y}D-YbSKvtW;`9G8S3QesU;A3)F06RQW=emy)&puQ&$}bQnaL zaSmB(Te{m2a^C_xe|KksoF zt~|r7y9{f|s1ez}VEO6|a~H&F#_3%{nWXPF8H!wna`pI1$#a&nK695Za1)xvxb_oL z3@36`?Q&iQ+eZ*;@~Pk94cG#Udo^NS>b&zQ_i&Z`WZGRr-$hw)o#Z8aZQ-9ah<1@M$6f>_gt({rgvSn4iO_F8-qGE$myFT3IFX@83* z`w?5WTuy1#@L}T50bd-dz@(s{ECS^=LxVFtJw1_~)SjN$C#!-ei6YYmR5FOyrANE2 zEM2`i8nlC9RlHG7W?XS`YYZCmLbDMIQ zY~-B2m60*2YmpRQ*gjJ4rjJ5#G!aFQGN@3Hi!t9Xb9hvUEJ8>II3TbS1r>!?28SGl zYdN`VxTtuCfzyDH?EtSLLP;pSe0+Sqe&tBl*}Q4f0YD!9r;{j(_;>6`em$&zq`SMD zPe0RH*5|lg+sAdfS+2S`RYgb)!nPf1E>okeGe^)UZJ*0w&y^zYC(iHy`%}UD>FG&< z7CeTA7YBIGyEB0)aiW@+8W^ee4=eun$#;sl7v8YnTiB0zbe4nTnR;S$i# z)IsEcTvST35&#zXldXl5-St`eV8?yHuYjB_%W`ugDWUj=zIj(ySAto!sP(w4*T!5B zL}7~l8dw)pbV2PUVW@v|AytZtAN@m|%yP{j+&^#LJc{5*M=AiD&5!}Wi;5T(|MEc8 z>{)m#0cl7NfCL-x%F)qLBLDM~3ik#zB zKoT1wP9g$|#+@-sPJi&=K~Qk8b=QWhsT$Yu^Ko&c4~7Bvg@Z>VZV{OJjs@poKgkF{ zpInGxSo)5dl;=z@uY$ajCr_5e8&O_3j86qd4!y}KV{WUB$69i6E(>WtWzk1fAMLa) zMxhW7s@&P+RtPG=1dDn-lqiu$pFh4Uq;Gz9;kEvUpz%egSD-3o#>B+D_Ytz#8W+yi z@(m9wz;MZehCr$U+@=Qn5uQr0_)L=&Y-f$=BkN>$I)8hDEhOwRW&Yq`cBb=O^1H&6OY#4%?(b4eY5~@$XZQHhyIAd0qe$;9BE2(IuPfs2I z`SmjW1B+04z&?*@^@~+|F}DkVE&+K_(@z1PJ{_^`s#1sI zP=Z=?5LBVbhlk?J*i)`3f0}Zh=zB{9Ii#)Nckd38a0D$=qx!Ug0Wa7=q{)gzvuZYW z_D$F%0Q-j#!OHO8Qs}tg<>WwWmq4i)e9s^E?$oxW$y2z`l(e+^r?r58r5R54UA1YN z$fG4x)}i!lG1gRT9h_8Rd&`QjW#98+mBb>pY*cShX*bJK-2~V}3BB@D(}x$}ObXaf zad2?7jK_H~nKoTjA|%Qnf%|YFqqU?ILIvdG;W_y8Mujbf4`kdjArX1syT{|^<_7Uf z54S?_mWq}|K(p8@OfF=Z_jPr8bOPP3pn8u+OgiajXKzoJSx^G*wgl`JQE9}egEJ?& z5Q-k#%9VVww5f8&l15UJPZ=1H9!NwOR|Mo0;xULj#6#d;yLJ~4^jP0p&N8UUdn{z) zdbS85W2>Dy6*5z?o;`ckcc*P+PG5!*c&}8~$y6YPGnq=dPmW!vsi|RGwJHingKF!#9d@U!ouoH z`hcRcGmcX%#N_}ADDByE(XOj16kH$4rqPRvOhh(>5R=8`<{u;w zZl%h!lt76TikNk=Lr5BdWT*^5v93lRtR?7hK3-l~1PjDXO~6c)3WihrGS{NMV5L1O zg)-9*d+bo< zP4jVa{X#@AsG+kmsVZ&OV|ppc$)LalDkcz~ul}OW3g99Bbsd_`tp&CYo-5SYrO1Rza(@YD}rwz8Daf|Mo1>_852>cr~`(;E>Xiu>1P4ukmfd*~a~A#I3jr2bxZ=1vnRR>CfG? z6HL#jPKFmYiqX`>sGNdADZy?q_C}D7=1Oy1kqTo$9WPLE@EQotj)V$55;m{0a>fF* zx5EYCyUpMlc5n#vHC5&U7do3t@pLw;o*ch~@Za3k6@y*Xj+HDz_0P6?wP}J2*`~k) zAV)#$HO41dojqFwF{KO%5&}!-Bm24{V4AW_7pJId6d2FIJrS9U`{RSUvt-EdIA`NkV=2Tq{+hwu-P61y*be_vnbOiW>DE-nK^Velq@XBFIW8RBA7Q9S*He^IC*j@8{4VA-ypyPV3aV1NqEvUh^=0bjFP_!5YY5>`fV31dzixU(}~C>r&(W*$fV@ ziHz?UIquc-lyuHL-qrr*;0&g+dM5NObobc=f&43y?325S8tz=C`}i|UV@?+`E!&dr zw*X_TD+!5Rn;$m-B*MDVob&uT9B%Rqu3HyKOTyP?zZ_`j*I$3#V^)1?8x@K6y4q)x zRS!KZ!Mi4}y3V%ib4HJ&|A>!9c*}nVZnghMa7zP`P8RWn1V}!$@Bly!GDiV*_3K(Z z|9g=ZU)R)j(anVhqOjl%#+}^5QZpQuv}Zw$UCFAL{tWI04U?gZ{ZU>e9aAB>*XFE7JMBw~hyMDbC>>+XKvUm5! zpfXwdlU1W;GH?xQTV6>?9cDZ{nv~U`xp_+V{|YmME!+W;BWb8G33ni|^$!lJO;1hQ zcGrl@in*pFpk|^N+qLPCFBfD-k`?h#tDYHnJXZ7<^vdM|`3=V)(8PP}2GpRGzE-bZ z4TvVgi!uj6`Yw1h5XEI7vOj|z&kgcJ=4p#bsw2#eS5hqlwjhxZRgD+Wo>ro11jy;ul#JUO4s@V= z)Wqg43);uBZ~y)duvHrP%Cv>CljvKrMN;&^ih7(>@@^&nz-ZwuKyy_6>Y2_C9uE6L zpUl97eNfU^GoTHT@=4keheOkCojez3=lByj3uKFNLu4KRpduOGWo)pB5+%?EAGEC6 zR9n@td)Z(lL=bO-lmP12CuCap;V=w32x~CJc#=s9xGWc>But>1 z*sCJYkOzdGN=r+(bREF0fTbt}<^nG%Wq`U6!I=~Zf>5ZgDI$_y?(m2tK96Mr#VD{N zaWTd~T~>B>AD$QHG$V75;Q=R{>pNiv;{pIW+?G*DXJIJy+o4pIKNPp#ENI0N{A6+z zYMXkNs|&$LI2~7RJ#iUuBs#I+{d>|_fM(;U*aWqY-3Mg;hb`p=Z{$iX#zqVD7cLT;HUHXuy4`Z*m&@xHmYkLN3pL`~=KFv6kt z&EYw7pP(25!jWeR>ZS8oSXjj9rfGGSN*UL5+!LWlL!6zJTaKNF9?T^R z%JmLhJlMw3Qwva(9D$eWjbs1U6BKLR$gbz-ph;2 zLXZY-`xvX6#7Z_eY4eI?bq}C6-3LNNfj`#oVCvbe z8)Z?IkAat_lLFvmj9O)GY#7e?vJzCZq@my`@r#Lx;jwlCj}hMS5;ONk>7{q>^A(Ay z6Z-d3V|ZuwUl;AaT}5lLA`#$?z|eRh0{99W1>v*E-v|4W`Gut8aI0WRWAwB1X3d)A zyY=K%@`2+l9yRj^xr5Tp_tmct{5Kr9*#@WJwxsGzm$Z;ouig~?2hikp!(_FFX^wU`}q?1^C)B*9G`w|IO7ADb}qZG)~iT%enF-Bf}+jW&cm? z>|wpVmc$qVw1F{ucJ7<*v10+T`dP7{FxVmslMtw%w|PJTT)k$EKuhiFNIh>D&`VNNL%Qz_$Umvfj3u+uV-cT)T!@`VNV22wo=M59{4%-Er*jr zz3X__8YmMj;+DuH>!bc1%vP7NS_Y@1MrseL3Fw?bIF{`oCrGdaKs-IgUGGa$9?~fC zAK?@bXz@T-hwg3J-PhMgQJFaB=VuCA9F~YeW+j6Kr3+-wAh;}DVErL8K57%Pcpzix zp**MW08R!VXBqW3<+hl88yh2=jvP2^tl;L^m#U>;MAb^M3WcB@Z{MRg)Ky&(l!0su zpL+l~HOQKciiCI_*tA+ITZ=;w=Y7SkA5+o*oSfNA^ZJ%PmLY0L*ocW(hVZo+E3U9% z|Me$TiTYWtWGbS}i)eq}vk^dQXuKzjupq4^(E`Z5CfJ*BJsgcdrpSpc!C%oeD+%@b{gy)ixYw_LA&C&t+UM%xHClf_1o#H6ZiMWBh()44?32bo zi0JU}hqxXMdD{@(8)Q=Bj`;X{hW0>)4n@Kvq1fhTn2UpqwYvs$AFUgbcA&O0) zfdDlK+Aaz%su>n`$hv$?$5#QDfmH7ZtXr03p%0k{Px`P!qL6}H9n4VB{`kQ-Vy!FS zOhs5eoj|1`S+Mv#rAvD=-*UlXgP@>+#*0CqJ?fZ&eUE1*G3zR!W>~Ob!Lx5r zCnJECt5R*F7O&YEf?6IMM}aKFz|N(yx)K9@Jo&%J&~44$O=;muU&zVQ(Gf{hZF2IF z&3XCx;FaI1!N^NZLYMizyLK0wcm4YHY)h6X7l$^$;RTlrtKUSdxKclqDf;@ezBW5D z<n`Y4*`4*Oq2U<7B%5(HzKt`K*;#e%)g$9C z57A5q)`%$gQd8%N+0&`JI{Qy~z$2SYgV=v=u__fbpJ0pfy_ znfy(umY*+zYP}Xc{Bi-Dwon(*I1zuMUel=ES~)jL1|_!I{cQ$vu}_?lb@q`K3u{^e z0xPP)hS((a($+y7zqs)5aOY}3bRNJ11Zs^r9_b%iQH2u-778YM+n3$T^S86Exmw+4GHoZblR+d^JL{J`iIN$-C>;3zn%}?uPiP!)Q2)d42 zHqS>$6LB6fedyf=Paa`mHFhDreCsTT7n-2ih@V$fR3sxiVxN~yqmQA)Za_d*@i7-2*U|vE&LBhM+0ggKBPQOKMq7e?u!?@WhMu~<}*v-{0Euy9&9lw ziSXKN?!diWBi$Ue;XAz2)zy_cP6oQHl|A4(LAg~Ps~d%21zWo;T;DUUSBR~kCL+R)r(pKhw!3@C6srS~?&^*h*^^>fHJrR*T zxf;e7#M)TUc@VZ-9lKLnZF(DC%;xlb=~puHs&=M^V0$#r;;XO{gU%YNnVCM3{^8+r z>@0(mUJ~=jp9h@F1y%Kx^RzAMC6wTg$yN+XjTw@io$epD1<4gw6Qh@xe`s+SXf5;s z8b{iuei!5?ay7=K_g@3BjD!0q9JLCQ3!Yk(Ww_4Vj<_dONkVeXFM*Ij5}bo$P4Zci z56Hm_rvo3b5yH5-g2Hw14BWB^>;Fw~#-V$>#pN3ytOixS`&ZY<{zu{yd&lqE>P+1G zqrC=d2tW`)RumHY;p=j7lqx`S#K6)#^tDozdc>}=~hA7MMQ6c^ADHmdtd<&}U*b&?B3YS%LBp)vRXm zjoWj8zm-5|#GlIji3oR8$BC&KoUY$?;|H)=T|zDm$saFlcWVCf zB`7{VUS~iRonRdu#(si-)FSKI&abgsBFX3nK<^VkY>q{8NAggr)r zBNWzX^I-J3?X+w-PjK_X>w>}pmNC@pH&%rwz5e9EfaQ!2(qB8wYoOlGGnx_xAzwEY)<-vy=5L`som_lRobfQS9D>3 zuD>{ZKqZ#t1KI_|3)gj`wk-v`Y5M&6cBfOu&`VL&fWZem%Yn&ihR!wd<~l_~GY`)X}i%@WYR&mBCn6N(#Na zx^P7h;!qUGyRl{;ePTy@vZnRW!lVXw8S>%~;23RL>VSfUBHp528Z~!i`XZ3_T{S7H zI1bc6x$l@|8Z|gm2?WJ>xVttAdX-lE$k0&P`B`%)9~pz#BZC&m4^S?}lH)qQ@vuLD z;VFf3NcZH5EJ5KS5=6cej5(LhP9i_r2RT8p5i3q`ac9yj${!kswuL88YHIMKL1Go#E5 zq`^@@I!J&Tpih=AUmpJJuk%SFfAeNPHMGIg5{r;?kO|#uD2H9YjdUy&nrLk0&PMP8 zmqDGcIS;-w7c+RzM!qv>yheY(BkB+>VmRfx`UFt}pHr6=*=jE-%r*Ijt zB^P-qN=M+1NuC~+j>y_F=>X_P-P8dAYsn@-o=1e`vP3`?!Vx%fGr11=&apkz2jP|% zXbq0PcDOCkp^-+nNnG8#Tk9Or7|=7iyhiY^f)xN)=&u*4bOH01vk5Ok8l$dATX?Ik zT)j&C0~stxn?!vqTkJ9uGT#+=t1^3n_WUNVsOS?F6$L!n^ySNmr<-9rz`d&il&L{X z{weZD9xd=1qg43lY+2WA{^vlIE&Pa&wm32#4(Pmu+ob6qkaPYEjdQp2KuQ#}8bXV~4z*LP z=I^j}#O7Nzgcb`C%>pdR|ALf%3P`zc!j;`cwTkqCjP!H_pA_e6c@;bdlw3maIFe%y zKrd;>_xl_UqfJ}8l=_gPT7s;^z~fx24vnhlpEz~epx!6ZC+>7><#2yl8k(t-9D7|i z&Ru{>A%7jUkV_sS43hnNe8?X@Iij(s8v-6*dBi>p(79PQ8S?mXB&aWPON+oxsf+0% z78yf#sm~9k#0%%MXqm}kS_j3i9rIP=zn94ga(u3`GFi%ej#MNrQ2G{ zr}hz1p8NpR#elB(kLb}1MH1-%RYuY}SNj!qy^r|DAiBUJVKk=<2A0g{jK4=7cscsE zG2}ez6e^9;79$Rn43A0Xk3dk0+4j`w0^6#=8pV<^`Et0G#uXs#$H0RVB~`F)@+)+qfLgox9hQby!ZWn0$g*N9bg_Ot$6A@3l%pCWt`N0U`TB(gN-gWG7`j zRxFzc{18xXNTn*mtOXI&b4X^g%%+YGJ`fE^K1U~)g5-R!T>uuLG z90Qv8fR(5~X(-}2Xh>)tCX&(E3iNx^I1{G1b3L4iq)|{u(Cjrc$aJrxqhp}wznYT3 zQTZaXbG5@=OU^#@P&wfl=Q(e++4N_~iH6Jo&~y|9yF=SOmN1FpVF)`DYKJxnF6fW> zLjfY2Ha&_N+n)an40r@(gk072E+da%awUHyok*0Gx43BX$?xdtSz^j;37{vqV)9F1 zmsUkr(molG_GUDa#=-p6zkyK+pRR)8h1O}Un6OL@>Z$di)*;Kcfa4v2Mk#*sw}9SI zyrbsR_(KW;3WmE9-Kvhc)zuP(lUaRlR<%sSEbsTVt81tuD$W>_TyUUy)M~FdRPO?= z7=KY)n?clbxBK+xxP?yv$yH>KX6GUuMZo5oTBvTFTi>vaf4$-H&g=wQvHhSi6V_CVUp6%?x9$QwD#A;Ib{ym@R=*VUST$DP`+IkLw z&t}@3Y|2I~OE~`Ody*o1(g$vmY6_^#ZpcXtB@P@lAtRcw8%-wtnTmAR#P#=no`RAQ z#|cKjjvZ*o@z2dED=(@`JW@3~JLcTSwF_UsF^;;_M@TzMs1P1m=4K8Xd+Q9;W_&2z zU@yB3F&22*z;+eEAz0#@Iy?DgWMn-4VL;Z`n??u~=6cZGylVaykYdjivBMxC4x4#? znktW_*o>%jd7LCW=;a{b4c)lf0Xsj4JgWG+EP-`TZ$5w|bL!;D4UnGx|Eeohv|ZHx zFBO)ML21a)z^jc}qb;ilgu|sdvBtem2x<=wgC-(6TJfY1t>od25cor|fV~1*_LFZI z(xDQUG3-kc7lvI?LL@&A#FTi$qV*Z0Ef4qpl$*-kb(~v z?BI)eoTQRb4;w@zHfMPJ9a|sSlbR)CtsoLo z=lQ+|aDzs{rR|6mtsZuG&7zgU?!( z$C&H@OI`?gLldNm1uV~9d&2*7qLO-4wKYl~=;EaGp)t*^WFDt;02y*%ib-zV3fEsT zeu_*W1qn)%`uTGPIJ#{lVU=cALr#N1X0tT3I`X>HT4Gswp$1TV!7;>~pl#%_{*!T2 zL7aP3kk32$PYqm$Fg0OW9(Xb$un-aVqJ$F&TyPzd?7SX=J3?gYiAN#r|PIEjg(dM(waJqVrY^>5$L z_ySq^XeQ)gS@sJx=-}r^=rXH(wog%N$h%Aw*{&S(QK*q&Af4unVJmzjJvm?!*8G3p zL;1DV{(G1QsTZubA-BhO4!7s+*`aFad*p(3LaI=dNI1+c%1QrYRLl_*r{#alB>da{ z!d{R!@kS;6W~wVYfqi$ipVlm%yEX}4cYcfpLS0*W(o^d0nKV=fsJBZ__#ufy7roQE zlibnZrt(YL+uN_iT9h|%F39+hvAo}*b58%MaBCjnz&Ha5D)MPy z5}qLTag8c)oH5s!?X&txEA1C`me7)U<8|u1d26k=YpU1 z?xRPK{9J~Q4p_n41czYmhwc}7Iq1Ddm6QNmB0l!wrAzXr(^r~dZvlttW3yrzT808q zy_z&+Um!gi9^}#_i+ZQJSlHU&)Vmh&a#hl%+KKJ7HiQk(Pq=J=qdbdz4c9NDH*^Jj zOeAq}x=kp~MYvDA6LMlrWqrgP*0IEs#zuSNjR|O-VJ^wwO zuR{5Mw9dESpW$HCt}RE56F%{Jz6<)J(4zSS2nDUn0Vq+MwJ&kesJYQ{m(HzG%E4E; zctHhYy65U9ws;M!>45(_^0WS~^PKQQXZ0Er!Cw&(s6hkU#0NPF6=le`Z{OBeisHfd zAK+1vgM@4dS3BuHG~{YPrfr}EV?5CNRe3@W5)eU;)}0uwK+fxLl3!tM8N|d4Rg{@D z_r^+K1VkT>8<#ASsRj@~+@-NlCx_Zg4hUs_3tG$}8BU#GXrFSBdazwYL>=LfrjH=? zP(k@f+NH|*sr|qgG)qm%$;+33#}z)EzZ_jSiqJ>UE9@Anf6`)-TNIs)fa-zthB)fP z3Bbeg5xbehaY_x~Z%Z+mDZewl_azJE7wO4CRcfpt%l-4`)_yL6g48xhzDNW~6ZBgU zK?r1l7P_6dti$BbqqcplR(;tpOSM0>G85<_a(Tsy2y`*TwEr&R%6a#qOJa6o>qOutavASR z8mq{f3MJECDw2h#O`n59uK@$i3AFF7(Z&r?_LWq(7C z6p{H@L}H|;^1Au1WBtbPQPAK%DiR=iAfkd zRRq5G&b@o#AV<&`^=4w%d-RcLMY}oqd&mO>o-Xvm3yo!&L_b6O(t;R!7ji0qA{zoe z#V~Ap!NK8yvBd+$B#gfa{Qp4RI62Q*s=NeI&f99leD(Q=I(aR}ZP)fVYh_?sr>x)^Vh{3qRdp)oAv8dWZ!r_u7WA>;s+OnJj zjWhINUks%AM^p!JwPUJ)z~^feY%d|NzQ$-X=mS(sq7meQ3?^f4O(LBTPd>kddZ}n^ z4NlJ<+Ix7a3E1$?oQDnB{#rksbm6+%attWar@>Xwoyq|BX+9R(#@{{v7g!FnL*+z@ z!)-5Ui2t;Hy7g+QeY+GO>R4M^Ti2mQvOA*uVycVqA)s;UT#Yh=02Kv^B@zdxWhyl- zjUT)^db+?`Xrh6bOj6Js2Y9^Fb%4;AgPv{^d%^K(!f+Cg$x1jvC>WyObtFE1l29jP zk=}QeT&#@?ST4dS08oL->$eds10j-i!YHMPED z;a0;m1@OmbmMpL7Mr0|&rn&>^9WgHw?X}CgTrkcEJZ8qXFCH|S2e|n$QZO1=gbSs8 zHX_lL!y!&tljiV|A}{S6yJhk>504T|RX`haIK(lqkp^V>FH6K;QT=0HD%q}_S0mBK z*1JvXLJ%tfwU&S(pw!p~BQ+`2{gR4s5Yy1G8Gk2^6L~jh#xi&B=m^_lTffl4T)KA~ zMx=dASXPQuTaeUQGr`5p9Rb@6tXnj{V-Qq0(RRLQt%g7AF^s$P|zDT}Sr;{B6&=+hvDA&B|y?LwSmk0}=4IkT(); zNp<(dYntGhv9EJDfW@V%KgK*5c^4C=RiRtc{3C!jphn6Jpw7n{F0qm>TyTFo+LDhq z={xsbA^Rw)2E{kDXOo!%r5a?~W{HQ+|492@gyH|E?Ry2*(@2p|{hYeR$yP@mI1m6d zc_tiQwowkRm!)Qy(?*^GR8GpEFEA^&H7zn>EvS>34n79}_-1tQU_bEk^ULEjsKCfL zqhsK{TU@HDsye{Ulww%GTJYHr?d7qDHPzHgrOGft-2I>GJ%!mIn}*(eNP9z7227VC=izN7wY{4Ss)3 z1O9J+$&8kbW|3M!-He%OP_hPjpO!gQ=V?B_6TCXg?9jH{qN(nx+2@O9#^xU)t&fb$5aPwg3ms%5$CTquKqt3h8V%0hi51e#=`exPjoR|x^SJ8lrDyx0ZDjy zc*x88pov3n9j19(3MIGj)F!*o@{Lk!rOXPCD6`{&O(|{j&R=Z&m-qS05qf_%#%5;e z|MsMt|B9E){7D5c2UhJJ36egQ_Z0ZF-z0vj*30ybS>1fZ%;b-bRmL&N8PmX(AA7q; z90XY+bY)q-7Zp(yTbU&5^ZXe)kTqWE%Ile#`?-Dz=Cqyqq8??|scu{58>o0MY;Txp z5od%?OrlHatD{>j;?C zjawHZ|3W#ESQ)!MTNWs%p2Z(hVC-5qL-Q-`0mD>ENw6A_U(}meQ_7Lq;Q)~-qrOq`h0aXYuv>FmHs;THbG4`+y zfhf8sgAoQfYU}7#4aZM{Piwdera20YB3JYZF^>o0@jm?@XMP;`_TYtiItb37vSTo! zjYm+h`({@hnpp4{J!lpRBm3U`xg1J>qGPaWQE^IsU^F7YbySXd`vbZbq=MPninxJX z>sHO7-#K6W1_#bdwRm4P6QjVvl$8Rj?TLYFjvsOH5U{GPpBeSapgO)624aso(|YTx|)xE4cwa;|(z`SyZk{AlgWq#O#oPesFso=%Ft~ z-du+PMAR-pQ;R+!*5hmRGZ?|Qdx=bAux*MJmlN~*i)Mwt%ilw9@ZWPpEIr@cToVP` zZ?mr(CUbxWP5h1zpaAQCjApKhbL+mol#i^3E&_P&pWK@Ut4zbefh)Pyj=uOe>@sh= z75%29#Pmb?fc5lq#)3ESC!JFwa~UdU{10{avKPBhd;xNy zf^7IHdS(}&pIEY>CY1hU4CZDLsYHLs0;W#A(-sy&fSY)z>EH1qtuh#E-7e3zr@0_A za~NsH;j`zBN;5aYu~|~Jfdd^4dNdJvDH?Soiz|i4H z6g%9qWCuZiCQY3J^i-f3xSB7^X7t8nJ*U06%)~+P`PTbtiPz1ZCS^E_Xp29ax@zF1U=^R0hF10EvjVfFFwv<9Xoy=D zTdNp7>in`~qC@E=_e>bA{kqv{^{pwJL+rQ(mVYXhtTn9JsQckP>eQo)>;)W#+*sRO zX5xI%P<3y|KNja19Q$jeEj`IqfJ(5F`hcNzVF+g7K_<|{Mn%QKpJHc*%$`rbkB1rU ziMiWffl3gXN`P$!->kA|L%Hw81%RqRW7#p9ullFhEpFiMX?`Ju1b_m$)YXjaIaevA z*uNAx(7vBXNdae)27c2#HXGj2FRb*}tImIUji0{RKKI%?usg=CBFCYDVf^KhmrH+N zd6_*IoNQY1R}6L6YE#=1#568m0A0`{G<^zkncB0tGY|dNJPzS=o{;h$qm7%o5vd6o z!?F|5IFG3kCTK6?*5#Oa@D5+@&mMUPS^&$h0^{iBmL3R+C~?&>;4PO;V^>W*<*_ju?V#l!MU_O;3nhp`f&Aqy@fHfpJT6sgg7A*Z;bDUiWQy{Ox+`VnCRY7m^u* zhD|khhalJ+(wsk0(|gpkO0yyXX|!d1exSGAW2O9p*{*{yD39n<1xgi$So!4QiGLjvqQU7q zpKfC0qRGwTf|)2o<6D67=nv@#!yQwbuR$G1FJEE~X|xLG%&QsKZzwfz5Riwhwgf!^ zHW>eeiI!~u+3L`M{-<3o z)KOq_Pz&mqnN0?Ccv^3EzW(z+IpdG%ejXWz>GHE*U4nR(j7eG@m?#F?Vrlp!h-Hdg z)LoC3K4|S9VWW-)Qp>!@k3lKOOPjySr7~(T49SVck!5!gp+tf)Xn?z1Wii^C)F}4& z@T15lPo8|?BjShA`{VJD>^a7`F3461_t@dFLk;JxG;`+9&s($sB@_rDYE93PM86ug zVkd}O8Z?z!2kwtZqX8ROt)7<8oIjq7<-EtXz%A;{{6lepY2=6+9(7*tRLt(fzZql-cBh(+FChA+$`vlA&89uDvk(Vu9^?8IM& zK^t4(4*8kGB)OJm51%@9iU++gSQDmsy44o^ll)dTjqFnx81bU!|LN|^!*WjF{X3R1 zVg{p;tuU%7%2K8z#W0po2}RnN6oqJ)QW@JXOL(bNl&wOWByC!3la|-gB1@ZAM5&M@ zozLfu8FOZ?bItFMbFOpFd-;R%e%I%@pZmG*&*F(f&^zc^z4F2$N8%_gA}D$S9E_4r zv-slrjhwM{(f1$#fJhAzs8BWyh$;LfKeaJ^tPXf%6MOX#lFzxNwbYe5(o9{^EkrYC zfAIiuY>>$Nm^dD<{;zTd-reV1&;P$u>9CoO6tewIW+|JM#iBMj{&C802!kCvN5kk* z&?@1nj?x4STw({12I&5mX+T1<+dH5Ao`r3DmPTKSURq=bF`Shl-yFLz%j>lHrH60C z`b2kgrk2{&>Q8Zyzh?XB4|yUeCjDHZ{qf0nE%S%AkNV47LR!aN^rzk4@%!BDd8z)f z+52o;{=Z>6{SOpsyLQ~D6bz%TuUQCm!duUNdwdgUWa7^sg8Khs)c}Slex}>Sf7yAU zv3XQzV!FOx#mbN3P=6=(|JQ57wj7_E0T9w{8>j!_WnlY~-rgtwtwa39Qs6IV$-(l< z)5yrCA1tp)o!WM7b;Dvg$R7TCB!|N)V7v&C3K2C!aX<8B-7f6|`xXerw}+KY-e?N9oA-+o_lDTN3FJDH?cmJf*d7o*oTE?S*;st z!dMK7kRUgkl2}W5P~>TpO2FZ(Q$w-$RnXEZtrQ303G%bCp9krP#bqQc%C+ur4?{9% zR1D3tOVmVQTN~P&p&rdXlzqcN&LA%|Dh>o-1fSiPNfbIhSv2(h%=Fw{9%b*e4Vyb5 z*|n?2rZwW53K=A&U))3UJ`MeELNbt0hGFf2pNEAUJ9AJuZn?}}D%K$` zKU*&yL>t2k+K}Kejf+v~dj11&``@!=M{x@9CnK?+qaP1oPis7fU7@Zl@^*SB3QuDR zr;Jko0J9KVevQA@qfbZh{`?EW#8=&`$BF5R`S6b;-h}Q8zzk9EcnzBRwY&NOVlzWc zt#eq3J+z?tzOgnhB69AG!RYnn>1Loe$rXu{fz0lo;4ct|uyr;D5bee*hodcQ$3z;IoCaDpg2rTz4NwcxS&Di_9p`m@f>>D|M5~s!St0AK7}LbO!LCrThi;L z{Ub_4rwAm6juzj}=Zf6olkff)Cco>tLkwvrzCC^8XX@0auYXl8P%o^AddyZMo8!qYSqIw{HZu`wAAymbofV558C# z+g-2%Aa2yW)&3OX1ert4b^deGFJ3`G^?EAf-Vh6i+&O#y^e$zV|0$lE506@`-udN~ z_y+w^NHS^MQlb4=2ila!FVf>Bb0h{tm|?}$Opk!^Y5LCSbwM^zJOKOdKFEx!Uw@uP zQjX)td(rk2ukd1g`06t|*EW`2I~mo|xj$p}3c1-8QGeGq1-MmF6^VJ6w8;Kll*s9+xblP|I-srHCeyf`gpRtm;+ihEW>zrw; zr8NRo97ngd*qb?;<#iAh!ak9F{Pa(P$D;p>&V+Y3GN zd$v~N82(p2K0u}2AE)x5S)aam66eInd)D1gSFuJM@B}LiN`(Cd-kLw(uRQ;K*0ha_ zQ4Lw>az|3&JxsQ5Thz^_6D=}hnk>J}IG=K}X>>t~|J#XtOM2v0;8lokCTa9{ntmS= zEcTRL+16X`$d`E4x32$wEf9kA&bL_I*SFX-ThYVTRU=qe+gxUx zMQhzr(;+I&(E+1ps_B?t(C{})vl9*5zw(f7x=TNv(RFB0H^Bn?#_P^<7ea5Q7{2S|A}w}rOEE;BwCw6>sPzP9hI#!|oN(5aql z`{bpMau?;KKYTSKyZj*UVB?Q5pB@dKvbW-!%Oz*;#8p}kFwI!65iGJ(rI%y|N$0)v9`}t)FC_n|P^@q^oDg={~c*Y5aP<>8m%UK6pHt zLOkLLv)tOFJKrhVJ}QqdzZ%nh#rO){53h@*l8!yc$*0>~wYVf$_~mxxZ^k;j@2*~h zpJg7~5iaolUJeiczVpsoRo}adfA}vvvfRh-;o&N%P@mM(yZnTbr|qp_tN6WltAG9= z=b=R$ID2iw11w5=WVNh*nDuN|v!TSY7AZqfg3U|05n004hMar)GMyX_&UF)n!|OIF5PucYUr09aX|U&h}B;D zZe~4?X&-pEOTAh71qOvp{pt_*?lmPudy6}A3di_vrgIQ1R@#s|&LPjJ!S+N|aZ9%H zCN1|bHZ-T%EpyRq<0+5q%BVJYwvq7!!?IscHwEXUCx{$~GQFgRDp{cs0OekS-jUKl z4>QXWAv(0=tFXWLk<+-*P&KpeRbQ zeIf;6U$Js)Juuz0#ilq7%dM!lQ6h@?|KLLbQOz#5XNH;hYD`8*X2L|1O)Bgq8W0f1~)VNY;364n3@@rYZv9r%y>5O)d9V^Y>D!@Y0p9WkxU z#^on-^AH-^p%s;59e=N_k)1Q31c}&y=JZRm-Z?{U!)q_>4mz70b<)~)Tx66{e9^PC zgDVFEtSJQfAkX*LZggTihyK_w^rtOy3wsm(`;Jo&EePg&r7@a?f2-vB}Uz=Wen;L?NKXbmOVh zul0Nc(c^WHIFEM{PXa+q)`G2zRv|~FG9E=ufeNy~t|%WM88jI-BtCGj^!Z^4jOf$e zJX|Y0P0k0B$| zEL+lUsRiqT4)UXBP;PZ??HGW6fG6gEWYv#TNXbI~4M-vcEK#@xJt26b)}!t>Nz=&Z z9fTvu2Zd-(wskzzz8=_=Oz2B{ASw;MeoZ70qGBm8}US)RgM*u$ei`4T2;PM#;ni`&x2ewLj zu}z@$$?4F)IXl~QHIW3nO5%4Jf38;k|$Xid!4I}nkJ@B~<+ zM~QHP=5n88kG2cYWrM)W!iu-_f=JBhAgZ$fQ@0vgQizw7O}BT<7cMpxp`LGq3OZTC z>OVN{JNYaP>RJ>Egf2WiZbxi+b&&c*=q?8zxLA#cMA7r z+S76oNkOs3=^n{s1%bt@F6VeGm?Dz5^g1c|nmlo3_Sl$OG%Xx|^O z45Yupg?$u3Dr&utrX1Q=LW~OFHT}{mp#EIX~H)~kt z0!-*NsLITpH&12M%Iws*+IneikNXE!JugWvhz>M8duDU2&iwhRvO~bwa@JRnN(~Ge zZ<@4Xt_tDmt=k>quG)Tp+u}X|*@n^|&j5_BDm0&++|`yUo~ebX(_knAO_1BKDY?WE z2>mATFoOpo)~NPep@Y^YwQkYRi~!NxuT9Ye7rz8pIEuwkL<$@9-a$FEGrb&DQJws!(1WI>|uBn=g9029V)`bp|Dcl&+F|r5V2NaOt)b&5X}f3D1!TGX+m=+W%JATpRchy2*N5zgQUz@ z&#^yC(PT(+8i3>k_axY^I?|H{RV7u;p!p@v#b$J;09TM2P$ea%^~fTQ`WM ze_6B;uGzRDtPZzSFd?v$;R2uxLljp;zEz7LY|9!19F9)Wf&hVkDLTe^zBMW+y9A(f zz^jWtm86Rz(Q91$7!ZQ+8@l=cT_FX)U68a$jw}r;n82X9{0yE&Fc0c{a&V5!UGRg? zq5FD7y5FE$8yDyl#&i}=f;b@Fir51Z_5)-A%F8<*yL@>n=yZ$NsJ!h;vzDCn;;7NT| zv`)R4jn`rCA&sIQ$U|TVSWtTo;x<0nrkJhzLMn`t1S%|yGFGKm5%V{MT+!ZMeJl|& zV}a#l-w@+-m5XFRZ(xlP-Rdt!L4?@q!^Pe@^mXK%qqY@>Kr%*S*1D}AxDgeJu1De8 z*feX5Bj8fgX=nt}2XU?_pb-~{(Yo4YQsP%%eRV*VBo>=C=nzUIo$jFXG9v4+%iAb(Vt_gB{Fq!)#DpT?ULnD_&Sh%iSy0gt zbd0P!dWV-fItmRc?=>)N=<-EvC>60@eI_=*hCfzE0?)6RnEfu z(FG|}UVf%d%Dw)RA$iM3hQ+by`P^`)3h z)Ph1oZE;{naGqUaByPvQBe-vvR7Af_2P$n)-41aC_Pn&H;4p$rTl}fyJRr%xLsmk^ zb1aTVz!0!VispT=v?z>`0kmD(D}{tubrzd}szIYPwugym1wQ&UPzh@=I<4n81|na` zp3oP1WplCEi+$o|&O}NzGX9j`5j@EVeGyU07@LL=j0Ks9R;82$XBQFQHH5hghI}E& z1T8bl!z*pIq4G;yHRPpr9Nur0?%#jooLdJJ&XoZ&+yNV9ib&CD5tDXGj1+uuzkBe) zCH|Lk|C6>c%M;GAxrh~tQreKFFaWp-8Ib*)a-&O0q4mRpwE&N&V>+sg4u6JNw{ z@De1MV6_-9c<`G0FI-Mxmoppxv_~TyOF;>euXP|-7s6On4XS_p#7{Z927*be=c-_> z-rypc$TsUlFsG?DzY~RS15kOZ;1{9-nGun1tqNPL{Mj|T#9>?xQQO{!2pcRN`7CyV$Rh5-{0m~YU#Bw$^EsbwO+bV6%FRn4zS_Q@A@Y-}kWLz&U?dhpS zkJk88g?`sCzft>aqX!Sw$Y`ww^*5 z*$IW;$Be^erx^r<$55)41fBw6oT(VH9vtcK|R`o#1><$`sc%zThX&j2%#(vd{E zIBXA0pKO9qAxSUSnmnKfgWA6zRrXUGQV2sKAJwzf-psfWD1FAffaj^Q$P?$o-)~7l zxbA!>Ge8qo@gfwgxD^6esBq>Vq}duqX=r-Om1GlB>j+r<=QriL35 zs2JJ;;JuKC#aRN%9Yq$Wz$f(duhY(k!cPcW0_iHMmVy%1$c z?__iVd^;p3#gBq~3r;ps8*>Ldm)y8U!2KgK8=DdyNU+-$OPB z7k)f!jFn@uI?TYx=A)J~z&X##GcuqtQgweEmLLiyeuP$H#;wK9L)aYp?M66oAo%M8 z%*M?H>=qG-msF$(Nl+;D>g=Gp8x(J$IN!4askUC=p7sY{px%qJlOXeJaDj|Z`GNNKxAr*VErBkA@5}h8Ak^Bh$WY$}-LW;0N>Hx$@ z7Vh!YA#U>Jo6#@aSYznfrrg1Z{I2nrV~G2{vj6e5swHS}2$Z6@H;C ziG0Yvin03%!$x9IsuwO8%nj}jM~=+2QG!GjmQ81ms~#{QWAfpWs#WfZC4fewz>!si zKck2-Mm{Jz#~PD_lQ3`?orTOe5qr7D57l&87uWsBb1SL z5AORor&126F=KMGRq@|kdMwU7(DoJGaCq}$~F ze$){^DudghLpyCDf@H_+T+cS{QQ1e-evEqRfjV=(elVC^GVr7XwnE8ss1k4;=@T+De-6v zP&uXt3aNHj$7Wf7iAZ1NHIZWnKrYIfK+v}ylg1Enx*@IATqi?!6kKxi@kJ6Qv8$*{ zowo(CnL4Ym8i5GzTI9{k<9^v8;t9uy3X9R^)DilFw4F+t%o+BkkT zHef{cQ9%W2=&V?FJ=ORj4Jn(!8L-+evMxYK0ulQ5x1}3EEXIhdaN48gIvJS=Y-b`- zfTsZHxM^h=@#!Q*p2FrxNL#WEWMv&W~})2KLG0J B7;69k literal 0 HcmV?d00001 diff --git a/docs/howto/configure.md b/docs/howto/configure.md index 8be733b66..b0575f3c3 100644 --- a/docs/howto/configure.md +++ b/docs/howto/configure.md @@ -112,3 +112,7 @@ Additional kwargs to `compile` functions take higher precedence. So if you set t * Enable promotions in encrypted shifts instead of casting in runtime. See [Bitwise#Shifts](../tutorial/bitwise.md#Shifts) to learn more. * **composable**: bool = False, * Specify that the function must be composable with itself. +* **relu_on_bits_threshold**: int = 7, + * Bit-width to start implementing the ReLU extension with [fhe.bits](../tutorial/bit_extraction.md). +* **relu_on_bits_chunk_size**: int = 3, + * Chunk size of the ReLU extension when [fhe.bits](../tutorial/bit_extraction.md) implementation is used. diff --git a/docs/tutorial/extensions.md b/docs/tutorial/extensions.md index a12747d1d..372439e96 100644 --- a/docs/tutorial/extensions.md +++ b/docs/tutorial/extensions.md @@ -350,3 +350,134 @@ def is_vectors_same(x, y): return is_same ``` + + +## fhe.relu(value) + +Allows you to perform ReLU operation, with the same semantic as `x if x >= 0 else 0`: + +```python +import numpy as np +from concrete import fhe + +@fhe.compiler({"x": "encrypted"}) +def f(x): + return fhe.relu(x) + +inputset = [np.random.randint(-10, 10) for _ in range(10)] +circuit = f.compile(inputset) + +assert circuit.encrypt_run_decrypt(0) == 0 +assert circuit.encrypt_run_decrypt(1) == 1 +assert circuit.encrypt_run_decrypt(-1) == 0 +assert circuit.encrypt_run_decrypt(-3) == 0 +assert circuit.encrypt_run_decrypt(5) == 5 +``` + +ReLU extension can be converted in two different ways: +- With a single TLU on the original bit-width. +- With multiple TLUs on smaller bit-widths. + +For small bit-widths, the first one is better as it'll have a single TLU on a small bit-width. +For big bit-widths, the second one is better as it won't have a TLU on a big bit-width. + +The decision between the two can be controlled with `relu_on_bits_threshold: int = 7` configuration option: +- `relu_on_bits_threshold=5` means: + - 1-bit to 4-bits would be converted using the first way (i.e., using TLU) + - 5-bits and more would be converted using the second way (i.e., using bits) + +There is another option to customize the implementation `relu_on_bits_chunk_size: int = 2`: +- `relu_on_bits_chunk_size=4` means: + - When using the second implementation: + - The input would be split to 4-bit chunks using [fhe.bits](../tutorial/bit_extraction.md), and then the ReLU would be applied to those chunks, which are then combined back. + +Here is a script showing how execution cost is impacted when changing these values: +```python +from concrete import fhe +import numpy as np +import matplotlib.pyplot as plt + +chunk_sizes = np.array(range(1, 6), dtype=int) +bit_widths = np.array(range(5, 17), dtype=int) + +data = [] +for bit_width in bit_widths: + title = f"{bit_width=}:" + print(title) + print("-" * len(title)) + + inputset = range(-2**(bit_width-1), 2**(bit_width-1)) + configuration = fhe.Configuration(relu_on_bits_threshold=17) + + compiler = fhe.Compiler(lambda x: fhe.relu((fhe.relu(x) - (2**(bit_width-2))) * 2), {"x": "encrypted"}) + circuit = compiler.compile(inputset, configuration) + + print(f" Complexity: {circuit.complexity} # tlu") + data.append((bit_width, 0, circuit.complexity)) + + for chunk_size in chunk_sizes: + configuration = fhe.Configuration( + relu_on_bits_threshold=1, + relu_on_bits_chunk_size=int(chunk_size), + ) + circuit = compiler.compile(inputset, configuration) + + print(f" Complexity: {circuit.complexity} # {chunk_size=}") + data.append((bit_width, chunk_size, circuit.complexity)) + + print() + +data = np.array(data) + +plt.title(f"ReLU using TLU vs using bits") +plt.xlabel("Input/Output precision") +plt.ylabel("Cost") + +for i, chunk_size in enumerate([0] + list(chunk_sizes)): + costs = [ + cost + for _, candidate_chunk_size, cost in data + if candidate_chunk_size == chunk_size + ] + assert len(costs) == len(bit_widths) + + label = "Single TLU" if i == 0 else f"Bits extract + multiples {chunk_size + 1} bits TLUs" + width_bar = 0.8 / (len(chunk_sizes) + 1) + + if i == 0: + plt.hlines( + costs, + bit_widths - 0.45, + bit_widths + 0.45, + label=label, + linestyle="--", + ) + else: + plt.bar( + np.array(bit_widths) + width_bar * (i - (len(chunk_sizes) + 1) / 2), + height=costs, + width=width_bar, + label=label, + ) + +plt.xticks(bit_widths) +plt.legend(loc="upper left") + +plt.show() +``` + +{% hint style="info" %} +You might need to run the script twice to avoid crashing when plotting. +{% endhint %} + +The script will show the following figure: + +![](../_static/tutorials/relu/configuration_and_cost.png) + +{% hint style="info" %} +The default values of these options are set based on simple circuits. How they affect performance will depend on the circuit, so play around with them to get the most out of this extension. +{% endhint %} + +{% hint style="warning" %} +Conversion with the second method (i.e., using chunks) only works in `Native` encoding, which is usually selected when all table lookups in the circuit are below or equal to 8 bits. +{% endhint %} diff --git a/frontends/concrete-python/concrete/fhe/__init__.py b/frontends/concrete-python/concrete/fhe/__init__.py index b6e99763b..e46f2dec4 100644 --- a/frontends/concrete-python/concrete/fhe/__init__.py +++ b/frontends/concrete-python/concrete/fhe/__init__.py @@ -40,6 +40,7 @@ from .extensions import ( one, ones, ones_like, + relu, round_bit_pattern, tag, truncate_bit_pattern, diff --git a/frontends/concrete-python/concrete/fhe/compilation/configuration.py b/frontends/concrete-python/concrete/fhe/compilation/configuration.py index 090bfe27b..52a2095b6 100644 --- a/frontends/concrete-python/concrete/fhe/compilation/configuration.py +++ b/frontends/concrete-python/concrete/fhe/compilation/configuration.py @@ -928,6 +928,8 @@ class Configuration: min_max_strategy_preference: List[MinMaxStrategy] composable: bool use_gpu: bool + relu_on_bits_threshold: int + relu_on_bits_chunk_size: int def __init__( self, @@ -981,6 +983,8 @@ class Configuration: ] = None, composable: bool = False, use_gpu: bool = False, + relu_on_bits_threshold: int = 7, + relu_on_bits_chunk_size: int = 3, ): self.verbose = verbose self.compiler_debug_mode = compiler_debug_mode @@ -1060,6 +1064,8 @@ class Configuration: ) self.composable = composable self.use_gpu = use_gpu + self.relu_on_bits_threshold = relu_on_bits_threshold + self.relu_on_bits_chunk_size = relu_on_bits_chunk_size self._validate() @@ -1117,6 +1123,8 @@ class Configuration: ] = KEEP, composable: Union[Keep, bool] = KEEP, use_gpu: Union[Keep, bool] = KEEP, + relu_on_bits_threshold: Union[Keep, int] = KEEP, + relu_on_bits_chunk_size: Union[Keep, int] = KEEP, ) -> "Configuration": """ Get a new configuration from another one specified changes. diff --git a/frontends/concrete-python/concrete/fhe/extensions/__init__.py b/frontends/concrete-python/concrete/fhe/extensions/__init__.py index 747e42c5a..36c006888 100644 --- a/frontends/concrete-python/concrete/fhe/extensions/__init__.py +++ b/frontends/concrete-python/concrete/fhe/extensions/__init__.py @@ -9,6 +9,7 @@ from .hint import hint from .maxpool import maxpool from .multivariate import multivariate from .ones import one, ones, ones_like +from .relu import relu from .round_bit_pattern import AutoRounder, round_bit_pattern from .table import LookupTable from .tag import tag diff --git a/frontends/concrete-python/concrete/fhe/extensions/relu.py b/frontends/concrete-python/concrete/fhe/extensions/relu.py new file mode 100644 index 000000000..71b04eba9 --- /dev/null +++ b/frontends/concrete-python/concrete/fhe/extensions/relu.py @@ -0,0 +1,48 @@ +""" +Declaration of `relu` extension. +""" + +from copy import deepcopy +from typing import Any, Union + +import numpy as np + +from ..dtypes import Integer +from ..representation import Node +from ..tracing import Tracer + + +def relu(x: Union[Tracer, Any]) -> Union[Tracer, Any]: + """ + Rectified linear unit extension. + + Computes: + x if x >= 0 else 0 + + Args: + x (Union[Tracer, Any]): + input to apply ReLU + + Returns: + Union[Tracer, Any]: + Tracer that represent the operation during tracing + result of ReLU on `x` otherwise + """ + + def evaluator(x): + return np.where(x >= 0, x, 0) + + if not isinstance(x, Tracer): + return evaluator(x) + + resulting_value = deepcopy(x.output) + if isinstance(resulting_value.dtype, Integer) and resulting_value.dtype.is_signed: + resulting_value.dtype.is_signed = False + + computation = Node.generic( + "relu", + [deepcopy(x.output)], + resulting_value, + evaluator, + ) + return Tracer(computation, [x]) diff --git a/frontends/concrete-python/concrete/fhe/mlir/context.py b/frontends/concrete-python/concrete/fhe/mlir/context.py index 4a74deda1..784f9f0cb 100644 --- a/frontends/concrete-python/concrete/fhe/mlir/context.py +++ b/frontends/concrete-python/concrete/fhe/mlir/context.py @@ -24,7 +24,12 @@ from mlir.ir import OpResult as MlirOperation from mlir.ir import RankedTensorType from mlir.ir import Type as MlirType -from ..compilation.configuration import BitwiseStrategy, ComparisonStrategy, MinMaxStrategy +from ..compilation.configuration import ( + BitwiseStrategy, + ComparisonStrategy, + Configuration, + MinMaxStrategy, +) from ..dtypes import Integer from ..extensions.bits import MAX_EXTRACTABLE_BIT, MIN_EXTRACTABLE_BIT from ..representation import Graph, Node @@ -51,7 +56,9 @@ class Context: conversion_cache: Dict[Tuple, Conversion] constant_cache: Dict[MlirAttribute, MlirOperation] - def __init__(self, context: MlirContext, graph: Graph): + configuration: Configuration + + def __init__(self, context: MlirContext, graph: Graph, configuration: Configuration): self.context = context self.graph = graph @@ -61,6 +68,8 @@ class Context: self.conversion_cache = {} self.constant_cache = {} + self.configuration = configuration + # types def i(self, width: int) -> ConversionType: @@ -2787,6 +2796,62 @@ class Context: return self.add(resulting_type, one, self.zeros(resulting_type)) + def relu(self, resulting_type: ConversionType, x: Conversion) -> Conversion: + if x.bit_width < self.configuration.relu_on_bits_threshold: + if x.bit_width > x.original_bit_width: + shifter = self.constant( + self.i(x.bit_width + 1), + 2 ** (x.bit_width - x.original_bit_width), + ) + x = self.reinterpret( + self.mul(x.type, x, shifter), + bit_width=x.original_bit_width, + ) + + x_dtype = Integer(is_signed=x.is_signed, bit_width=x.bit_width) + table = [x for x in range(x_dtype.max() + 1)] + [0 for x in range(abs(x_dtype.min()))] + return self.tlu(resulting_type, x, table) + + if x.is_unsigned: + if resulting_type.bit_width == x.bit_width: + return x + + assert resulting_type.bit_width > x.bit_width + return self.extract_bits(resulting_type, x, bits=slice(0, x.original_bit_width)) + + if x.original_bit_width == 1: + return self.zeros(resulting_type) + + chunk_size = self.configuration.relu_on_bits_chunk_size + intermediate_type = self.tensor(self.eint(chunk_size + 1), shape=x.shape) + sign = self.reinterpret( + self.extract_bits( + self.tensor(self.eint(1), shape=x.shape), + x, + bits=(x.original_bit_width - 1), + ), + bit_width=intermediate_type.bit_width, + ) + + filtered_chunks = [] + for chunk_start in range(0, x.original_bit_width - 1, chunk_size): + chunk_end = min(chunk_start + chunk_size, x.original_bit_width - 1) + + chunk = self.extract_bits(intermediate_type, x, bits=slice(chunk_start, chunk_end)) + packed_chunk_and_sign = self.add(intermediate_type, chunk, sign) + filtered_chunk = self.tlu( + resulting_type, + packed_chunk_and_sign, + [ + (x << chunk_start) if x >> chunk_size == 0 else 0 + for x in range(2**intermediate_type.bit_width) + ], + ) + + filtered_chunks.append(filtered_chunk) + + return self.tree_add(resulting_type, filtered_chunks) + def reshape(self, x: Conversion, shape: Tuple[int, ...]) -> Conversion: if x.is_scalar: x = self.tensorize(x) diff --git a/frontends/concrete-python/concrete/fhe/mlir/converter.py b/frontends/concrete-python/concrete/fhe/mlir/converter.py index 48f8b22a7..a31cfc3d2 100644 --- a/frontends/concrete-python/concrete/fhe/mlir/converter.py +++ b/frontends/concrete-python/concrete/fhe/mlir/converter.py @@ -34,7 +34,10 @@ class Converter: """ def convert( - self, graph: Graph, configuration: Configuration, mlir_context: MlirContext + self, + graph: Graph, + configuration: Configuration, + mlir_context: MlirContext, ) -> MlirModule: """ Convert a computation graph to MLIR. @@ -61,7 +64,7 @@ class Converter: module = MlirModule.create() with MlirInsertionPoint(module.body): - ctx = Context(context, graph) + ctx = Context(context, graph, configuration) input_types = [ctx.typeof(node).mlir for node in graph.ordered_inputs()] @@ -451,6 +454,10 @@ class Converter: assert len(preds) == 0 return ctx.ones(ctx.typeof(node)) + def relu(self, ctx: Context, node: Node, preds: List[Conversion]) -> Conversion: + assert len(preds) == 1 + return ctx.relu(ctx.typeof(node), preds[0]) + def reshape(self, ctx: Context, node: Node, preds: List[Conversion]) -> Conversion: assert len(preds) == 1 return ctx.reshape(preds[0], shape=node.output.shape) diff --git a/frontends/concrete-python/tests/execution/test_relu.py b/frontends/concrete-python/tests/execution/test_relu.py new file mode 100644 index 000000000..2c0ea68e5 --- /dev/null +++ b/frontends/concrete-python/tests/execution/test_relu.py @@ -0,0 +1,106 @@ +""" +Tests of execution of round bit pattern operation. +""" + +import random + +import numpy as np +import pytest + +from concrete import fhe +from concrete.fhe.dtypes import Integer + +# pylint: disable=redefined-outer-name + + +@pytest.mark.parametrize( + "sample,expected_output", + [ + (0, 0), + (1, 1), + (-1, 0), + (10, 10), + (-10, 0), + ], +) +def test_plain_relu(sample, expected_output): + """ + Test plain evaluation of relu. + """ + assert fhe.relu(sample) == expected_output + + +operations = [ + lambda x: fhe.relu(x), + lambda x: fhe.relu(x) + 100, +] +cases = [ + # fhe.relu(int1), should result in fhe.zero() + [ + operation, + 1, + True, + (), + 0, + 2, + ] + for operation in operations +] + [ + # fhe.relu should use an optimized TLU when it's assigned bit-width is bigger than the original + [ + lambda x: fhe.relu(x) + (x + 10), + 3, + True, + (), + 10, + 2, + ] +] + +with_tlu = set() +for function in operations: + for bit_width in [1, 2, 3, 4, 5, 8, 12, 16]: + for is_signed in [False, True]: + for shape in [(), (3,), (2, 3)]: + for threshold in [5, 7]: + for chunk_size in [2, 3]: + if bit_width < threshold: + key = (bit_width, is_signed) + if key in with_tlu: + continue + with_tlu.add(key) + + cases += [ + [ + function, + bit_width, + is_signed, + shape, + threshold, + chunk_size, + ] + ] + + +@pytest.mark.parametrize( + "function,bit_width,is_signed,shape,threshold,chunk_size", + cases, +) +def test_relu(function, bit_width, is_signed, shape, threshold, chunk_size, helpers): + """ + Test encrypted evaluation of relu. + """ + + dtype = Integer(is_signed, bit_width) + + inputset = [np.random.randint(dtype.min(), dtype.max() + 1, size=shape) for _ in range(100)] + configuration = helpers.configuration().fork( + relu_on_bits_threshold=threshold, + relu_on_bits_chunk_size=chunk_size, + ) + + compiler = fhe.Compiler(function, {"x": "encrypted"}) + circuit = compiler.compile(inputset, configuration) + + for value in random.sample(inputset, 8): + helpers.check_execution(circuit, function, value, retries=3)