From 353432271d356ece1399dff34cbe42d96e35110e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 5 Jan 2024 04:08:53 +1100 Subject: [PATCH] Fix tokenizer + docs (#62) * Patch tokenizer * Update _utils.py * Update _utils.py * Update _utils.py * Cleanup * Add comments to functions * Update rope_embedding.py * Update rope_embedding.py * Update llama.py * New logos! * Update README.md --- README.md | 39 ++++++++++---------- images/Colab.png | Bin 0 -> 11637 bytes images/Kaggle.png | Bin 0 -> 9733 bytes images/try live demo green.png | Bin 15262 -> 14424 bytes unsloth/kernels/fast_lora.py | 53 ++++++++++----------------- unsloth/kernels/rms_layernorm.py | 24 +++++++------ unsloth/kernels/rope_embedding.py | 55 ++++++++++++---------------- unsloth/kernels/swiglu.py | 18 ++++------ unsloth/kernels/utils.py | 3 +- unsloth/models/_utils.py | 57 +++++++++++++++++++++++------- unsloth/models/llama.py | 24 ++++++------- unsloth/models/mistral.py | 37 +++++++++---------- 12 files changed, 155 insertions(+), 155 deletions(-) create mode 100644 images/Colab.png create mode 100644 images/Kaggle.png diff --git a/README.md b/README.md index 041975b04f..f24487f79c 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,25 @@
- - + +
-## 2-5x faster 60% less memory local QLoRA finetuning +## Finetune Mistral, Llama 2-5x faster with 50% less memory! | Llama 7b | Mistral 7b | CodeLlama 34b | Llama 7b Kaggle 2x T4 | |-----------------------------|-----------------------------|-------------------------|------------------------| | **2.2x faster, -43% VRAM** | **2.2x faster, -62% VRAM** | **1.9x faster, -27% VRAM** | **5.5x faster, -44% VRAM** | -| [Free Colab Llama + Alpaca example](https://colab.research.google.com/drive/1lBzz5KeZJKXjvivbYvmGarix9Ao6Wxe5?usp=sharing) | [Free Colab Mistral + Alpaca example](https://colab.research.google.com/drive/1Dyauq4kTZoLewQ1cApceUQVNcnnNTzg_?usp=sharing) | [Colab A100 example](https://colab.research.google.com/drive/1y7A0AxE3y8gdj4AVkl2aZX47Xu3P1wJT?usp=sharing) | [Kaggle Alpaca example](https://www.kaggle.com/danielhanchen/unsloth-alpaca-t4-ddp) | -| [Colab A100 example](https://colab.research.google.com/drive/1YIPY_18xm-K0iJDgvNkRoJsgkPMPAO3G?usp=sharing) | [Colab A100 example](https://colab.research.google.com/drive/1SKrKGV-BZoU4kv5q3g0jtE_OhRgPtrrQ?usp=sharing) | (59 more examples if you scroll down) | [Kaggle Slim Orca example](https://www.kaggle.com/danielhanchen/unsloth-slimorca-t4-ddp) | +| **Free** Llama | **Free** Mistral | A100 Colab | **Free** Kaggle A | +| A100 Colab | A100 Colab | (59 more examples below) | **Free** Kaggle B | -* **NEW!** [DPO](https://arxiv.org/abs/2305.18290) support. [Free DPO Colab example](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing). [More info](#DPO). -* **NEW!** [TinyLlama](https://github.com/jzhang38/TinyLlama) on 3T tokens. [Free Colab example](https://colab.research.google.com/drive/1AZghoNBQaMDgWJpi4RbffGM1h6raLUj9?usp=sharing). We also show automatic RoPE Scaling extending TinyLlama from 2048 to 4096 tokens! +* **NEW!** [DPO](https://arxiv.org/abs/2305.18290) support. **Free** DPO example [More info](#DPO) on DPO +* **NEW!** [TinyLlama 1.1b](https://github.com/jzhang38/TinyLlama) on 3T tokens! **Free** example +* **NEW!** We're in 🤗 Huggingface's official docs! We're on the [SFT docs](https://huggingface.co/docs/trl/main/en/sft_trainer#accelerate-fine-tuning-2x-using-unsloth) and the [DPO docs](https://huggingface.co/docs/trl/main/en/dpo_trainer#accelerate-dpo-fine-tuning-using-unsloth)! * Supports Llama, Yi, Mistral, CodeLlama, Qwen (llamafied), Deepseek and their derived models (Open Hermes etc). * All kernels written in [OpenAI's Triton](https://openai.com/research/triton) language. **Manual backprop engine**. * **0% loss in accuracy** - no approximation methods - all exact. -* No change of hardware necessary. Supports NVIDIA GPUs since 2018+. Minimum CUDA Compute Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) [Check your GPU](https://developer.nvidia.com/cuda-gpus) -* **NEW!** Works on **Linux** and **Windows** via WSL. -* **NEW!** Download 4 bit models 4x faster from Huggingface! Eg: `unsloth/mistral-7b-bnb-4bit` +* No change of hardware necessary. Supports NVIDIA GPUs since 2018+. Minimum CUDA Compute Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc) [Check your GPU!](https://developer.nvidia.com/cuda-gpus) GTX 1070 and 1080 works, but is a bit slow! +* Works on **Linux** and **Windows** via WSL. +* **NEW!** Download 4 bit models 4x faster from 🤗 Huggingface! Eg: `unsloth/mistral-7b-bnb-4bit` * Supports 4bit and 16bit QLoRA / LoRA finetuning via [bitsandbytes](https://github.com/TimDettmers/bitsandbytes). * **NEW!** Want a UI for finetuning? Try [Llama-Factory](https://github.com/hiyouga/LLaMA-Factory) and use `--use_unsloth`! * Open source trains 5x faster - see [Unsloth Pro](https://unsloth.ai/) for **30x faster training**! @@ -31,8 +32,9 @@ | Slim Orca | 1x | 1.18x | 2.22x | **14.82x** | Join our [Discord](https://discord.gg/nsS4V5Z6ge)! -If you trained a model with Unsloth, we made a cool sticker if you want to use it! + +If you trained a model with Unsloth, we made a cool sticker if you want to use it! # Installation Instructions - Conda Select either `pytorch-cuda=11.8` for CUDA 11.8 or `pytorch-cuda=12.1` for CUDA 12.1. @@ -79,6 +81,9 @@ pip install --upgrade pip # Documentation We support Huggingface's TRL, Trainer, Seq2SeqTrainer or even Pytorch code! + +We're in 🤗 Huggingface's official docs! We're on the [SFT docs](https://huggingface.co/docs/trl/main/en/sft_trainer#accelerate-fine-tuning-2x-using-unsloth) and the [DPO docs](https://huggingface.co/docs/trl/main/en/dpo_trainer#accelerate-dpo-fine-tuning-using-unsloth)! + ```python from unsloth import FastLanguageModel import torch @@ -145,6 +150,9 @@ trainer.train() # DPO (Direct Preference Optimization) Support DPO, PPO, Reward Modelling all seem to work as per 3rd party independent testing from [Llama-Factory](https://github.com/hiyouga/LLaMA-Factory). We have a preliminary Google Colab notebook for reproducing Zephyr on Tesla T4 here: [notebook](https://colab.research.google.com/drive/15vttTpzzVXv_tJwEk-hIcQ0S9FcEWvwP?usp=sharing). + +We're in 🤗 Huggingface's official docs! We're on the [SFT docs](https://huggingface.co/docs/trl/main/en/sft_trainer#accelerate-fine-tuning-2x-using-unsloth) and the [DPO docs](https://huggingface.co/docs/trl/main/en/dpo_trainer#accelerate-dpo-fine-tuning-using-unsloth)! + ```python from unsloth import FastLanguageModel, PatchDPOTrainer PatchDPOTrainer() @@ -262,15 +270,6 @@ Two Tesla T4s on Kaggle # How did we make it faster? Manual autograd, Triton kernels etc. See our [Benchmark Breakdown](https://unsloth.ai/blog/mistral-benchmark) for more info! -$$ -\begin{align} -y &= \frac{x_i}{\sqrt{\frac{1}{n}\sum{x_i^2}+\epsilon}} \cdot w \\ -r &= \frac{1}{\sqrt{\frac{1}{n}\sum{x_i^2}+\epsilon}} \\ -\frac{dC}{dX} &= \frac{1}{n} r \bigg( n (dY \cdot w) - \bigg( x_i \cdot r \cdot \sum{dY \cdot y_i } \bigg) \bigg) -\end{align} -$$ - - # Troubleshooting 1. Sometimes `bitsandbytes` or `xformers` does not link properly. Try running: ```bash diff --git a/images/Colab.png b/images/Colab.png new file mode 100644 index 0000000000000000000000000000000000000000..7a21b9429c5fe6113794fe0e9bbf812febfda2ee GIT binary patch literal 11637 zcmXY%b99{F)4-#qv2ELIY};yVHrQB=Z5z9>)!1p=SWVK{X=C%y0~8G!%+0s;bAPF7MK0s@K+{Okk=3;yIgq3?%)AcK&T6w~y|{g(&tMYNc9 z{@&dRO4qOX^MgmB(ZQU_++r6mB#s@(gDQn>Y~tQ4CDcKY z#QWO-p6o#bwI5OpITgDVrwO3~&+rci=Jb83r8zt1?y6`1vIXQu&BZ^9hFXTqd7JmQ z5lQ-5txkV@&)N_3f7ZL7YVTnXQ`dNCZ~;oFF}enb;F)=4g1OVjMBDi=##<$WwKb?L zILHv(s4P@>8wR8ii`fv~k+c88WnXPIec9s19k8b#%B+Fe#i1>RHCMi69Arb{w8jVS zBlgjK7lVL?@PWL6ID&Hip1!=|=?dGlLAa_mr*9SqBW#bTvaX6o;ArdGicJ7a(X&tB z!@zj6kAg|WCBc1F9x@3Y(GKr_3BwyVfb%q>ra|sFw+lDLvvN|$^JT#O&L2gK1hIl> zBJ=ke7NQ{ES0V!^Q+c`a*WJaCCsBWrdzhG~3lBC9HCfoOW8ILb7`HMX!Bz{_{>?QT z(jc1~iO6?e$S$}oc>V0#yS@%Jllb(nE19GOovn0iUP`7jd37N)6|j2N%2?dH7TXx1 zia+?CQo(|gL_5R~YY;Dxef1*pp@g&1P{ommYRwlX!z*_A=c|v(%_d!vwRc1oXAM4*Hoy^An z3G3g6Dybcd)c8x<7v58*hIZqO$_@^8d*7%+H}H@5dMLZ!1oTal7Sz zuBT23FRSUh!76xfA(&1g#Rp#3g(qzkdd~PfQKvpB|GKX=0XxtYn(;d=q=Ry6i!D8y zVQo5Lm5jhy&bLM2EmC((OW9lOz;~9)+z*`i8kD(4VK_$EnHh!f}PB-G}=lqMYl9g$!!um<(*n z?|mt`=hXK_K#*m~x9NMQ{~Z09|M9UPN}sW!Pc`hA<9v+zrSTAU5hx=^o_GGI@55^- zt&qBui~3CqtY3TH3h>eP?*(NFJ=+0ZtQ$sN?o&cD?k!<9Obx&#OWargy0wJXr{M`sKgcL2gdl3MJA04H z|GBoT&uwiO_t!Z%sj<(-6!)Qlt5ru}t*>x1g#BsEtoo^Bkdpeea6}UjviUyVf5 z#bc;O&j8=Q@yllHv&ava(+oehjrRZzp`Oh8 z+}xi&0maGuYPOxBbRZ+Pt`{SF^<}q_7Gno3LOtDrF3A+0=LTe?!GHVGvRMU zbR!B^Pn=O~Kc%@NI?u04(K9m7SI{+Xk6$x}P5%Kx3O;F*o_Sw4XDe9&COvrrmPCdCkDKijYEMq5u+7w~tgmJxLoz zw=hf-4a_wLEbvFD84E_5b)qVFqI$T=e#wuLjJpV9!n-^~{CDJlqn>$|w>UEv=skqM!NX}o@8f`oDBJW|vfD;NdXq=(VWsxQY_m~AU1B*A%X9m3 zd@q+oH$gg*oZQsbR;uuTPmgwC>e~+@%{aU2_*zvI5VKFV|=j6-V8z`+bihJP1ua*;# z;D(h6q{ijpX}jmOy5>&Yx!MjNTA11m@r&Q!Mnvq)c%&~bEgfwPXJR7m>m*t` z|K@fYwgP%hYSePHwEVyH?qIz+Z&7r)3JO$sRLTf6miB=tOr}eZLKP6z{Mhw#H<=}M zXMT4rOy$5ZXD#Z@O!@E3a_j-*cY{*5X*~*Z=aUF4CTpN+1t0DW{pI|1O5gJjv^hTe zW*giawcxc1{?xj_Uu;v-1fTIW9mTVM^6T7t>&p4VSX+CLs^s*l6e#R+2ZJ%4iPRl> zn(r)|HS-s$k!j3fT5EhTxs`d}>fb4Toe_@U<4KCyU}2_HD4i<^v)(({%F~(5BcoD< zg<|DL+@Zpu)2S=xQrV21Th0G~ZanYb!vPIVW~{SvwI0c8+c_mc-8YGNmM)TVwc04Z zJ=M%NiO-iw$g~(wT&mhjFE>Nk6wAwG=11|e3ceB77<6~M!1TEPG&a?G7^TDicE`OX z`o6GZ#Tjv#km0nf>Fr%W=PY%NzTOg0c6T5DIADWJkE5;}MrHCXuv}=Dm;>IxmEmPk zA#Z^DCs~=r5=bnks+)}xDJKyirJVZx3wD*Of}>}ZC(W9pk&us~ zug?kvGKH%}+3#dvW8p;$Gaw+@dZeq`68t2tT}CHjGk>Uo3Wbu!<5oAz_p8I+NYr|` zUtWIy#4V*aYKrzL7Sxp)l;k1c_eZ;VuV5%H-angXaQ`&Il0(g=t#RGoJ!S7URoiH1 zhCA>+I<=4+ajYAUW;vTDuc0*#V`kF8ADF{KRGyXNZ$Bfu}CskS5w z-)U=!pFZdR&uV*P&t4Lp_R_b^U{Y0?bjbdn>XBpGe7t;_;TQEnVZ8eY7f>LP3K8A{ zRjlDz=fvV+my=f7(e&}3s(*?K)2)g{jYya5UF9n*H}qn!tLK-9@4~Ah@xraf&U4E# z$z1+Dlm`a51#Sx)o+=43J<$ASjM(&rS+?4{S-a{A_#k#dh!?e^{@{xv4(78z7WjP( z2Al07vGnXkkeJeUE8AaFbfbzS0eOS_U--0{Qc=n1ifz=k;RqC9LnXb;h=_C}2SYS4 zU(fZ8m7ejB@>56lXSSen^XRZR{;AVLm;H*kdU4^(xCNFpsyy6J9%A_!F?nrXacFo) zNyu+N=f~N$s-aN71uUM61d+S;cLG9GHXU9G55Rb$D9{w$$*hqtno!&A&#dg1dOFLujE55O; zbKc?j`~1zogWR~)R+c(k{|!eGAEjF!dghQu7Oz3lvcTB38=w|Jau zlU}*{p-6gmEbeBYHT?!*d2%(gzlR<%++41871Uvf%&i%*5(!&ljD zv5>uNvwm)9#^GJZaA-AqD!0uIMG$>B<=7|bG}Jr6xU3crsr-QZLGLA z8#!vDxmE$L%Ijz*j$pReu`b|}+!3!C1&`nfhr2Bmg9M~%Z@U9=1H0U^~ zwTWb-;D4rO+6+MeiSbZZF4lwyFDF`AM&90};1Mb3Tu|uwZ~lC~bMV{LyALyGgqc9L z-0L>|T^+qHM?UdO(^hOcU9@P5c;%9Ndof7gCk5sK7h%pA!oP0vLAxc1Zn^L*SS?h{Htoa^8gfeJ`U}6j$QlRZXp4REwQ*sx27l=#_Zso8Ury1 ztH%9jGn9m1RiLj2mJBm z^Gg|HCQ(OH#Mgk#ae42l-Qxq3^$D$CauB9x&b5ZfHDGCdrY0o58 z>=?5D8_7RB9M*s5B<2;k-{>P31Ghi++uN+*Xh-@D3NXbi-8hGMiGg@iWRBEe5!_tg z)BAqCtfsbnc zLcw3beM&#D>Z>p_{~7svTJu^g=D>Q1pN71yGiXK}_Bhc8NR5GZ52XRe@r9{rJOI+So;ldbv? zor?~`Bdd+I#6TJa-B95lbmjmvzw^P{VnhnQh;ev{G7LmBd`(Wd6PeEG9MW6{NOJ1Y z-4+0brF3cZNb~4%+T&HOomdG9hzEfyzCu!r<5Ttl+9F%ZIFuq{Q!=&qsG1RyPiw`G ztN;}R#x%L2zE34`GO%*{Kjo=H0J3?rSnw74eJ&>tbnYpsymmj0yT@hRdDB27rs74j zb$9aCaY`8)rnK?l3>L5MAyThR4yPdFTi-n>GjDthv1mAWjrLA#3@df0Oph~8T;5gg zsFJ8Mdyt!z5NC*8E+F}7)?JpY)ULhATta{SXW`Gd_>P-VKX2uhM1sK~UJ?;HUm{ut zOER8_xu8`hcd7>!71E?XRs@&^Z|A~d=-SZ$%ftm~ScJO**i-`Mq(ZG<&rDnieRP!F zXHJ263}aqZmZ?Z9?!{tZIEt60#zs`)X>zYX90kcvS17kn7~-D2bol}_*q~H6{dKFe z?>};R*JR}d+7&S%l7AZ@Rn&iFFGu!#Eqei?VeF|7G_6`@_zSgUu})Xen3?0TQwtgv z#7Mfy!hgCfUEG0(;rdsk66tG@miYpr{<6-MLf%wi~Ves{c; ze9CKxW}>%DRr&06qHZybcNUE(8AbGWd!o-A&M zr17O4qiczqfVysKg89@j z)|Go^IrG#JX^k$x)g{`o-pEwe;z5`n?>b0^NbT5rG34uF##QF$A2+p2q$>d!u!K%i$chpav-bP z0;IaKm&5KPr`n4V|i$yV&iYL1ad*w7R3`$}NSAK>nd`YXJjB;O7kf14eW_C$2FatjD3%G7uIf=T( zxOzuEuBeXtGh=0X9T?1V-9g0PB}@|xs-x^37fLtBt~xGZj#(te<8vqzFdK2NdHT%J zPuSj4bb5XDDZLr1FK1#E2pNak_bf`C`BpP|c#2wOOe1Wg9%%L!P}i0$DnLuE5vY(v z7w65xcrVX~t)a#(3ztL$?_$BV6Xn0-7K3BL08vSE83(zD=(AtTU8rxsi92*Gg}*z~ zm63C+hftAt=*2z=Q0@HXLy?6v$1X9PI9pnIBTVdpP@w5T;J=c%T;V0aAF;2r_r7|S zzT>!}eo?t>rF=es{kfu3r-R422E9$=ufxwyzv938Aak0PSWHa6&JE1T=a}0s(az3 ze;pxIiI{d--PQ;W{h5}(FVhcB_j3SMj;Wtz05n`Qyq^@4DqA>&x_Rk!%uu4$#Uru` zUY=PlVeCPg>rEwc*fEC1as7Y^az~?~;d%^iPg$$T7BMEN9wM7doC)NTGJY|A@Y~b8 zgz3V0#h;$L?_CV7Si>)RgM`sPO#4nn>tw>eDPgs_y|Qq1PFeY#n*nhPJbM_P&rv>C zp~aUPd5{@)rBf`P2d$VfX%U9Q{QKr)SX53Oo{G!`fbbu*fhuinE0A*%8ffm; z?C%+$=Lo|AP(qw`9>{Aaa*LkwJQK{QWttgJRIuKmKD&!b;2XsQ%Zb?&6=#}#eQ!iv z$FmNwK}m9@8J^@zMfoPX08de+L6aD(L6MM>4n7+I1QODx#*0pCQT2$>DfAxAkeNih z#kGELn@0D^iBi0V<$isEdXIO{%U=6l(a@|2c-5j@P=4*MpYab z&o*M{F}a0#CyHfOuJNg>$fb;ZI|BtzHiy=!hU=MsFLv|h++tEQN6vNAE@6X)f4Q}? zWM~>i7QD8*Ee(s>A*Gm${8|L8HvBu-FSZmLI#A_*+0k-oSHUR&tj4aC237x(M{BJu z;q=@eC(Co~`(j()g~eW8U~dso$M={a>z&2cr-~;wy9zN?B@#mF;De$dzi%LR^sT4$ z()(-A16-|LHU~%d##&^b1xAyBqIVGfMf2}%0jHnZhXxOAKxFUm1&yZS@u_bw}ubu(mGq16kQ>30@`U746uYWQhX| zo|7}{#g}4kjB47ZSM<+PYkvXH>gIU!shOJKc5+u|Jo>Uv*%C?&bOJ*j$3tn=;&XJT z6`ejIE5oiN1AC0dpYoTD7xnx$;kP^u0`y>|w%Xdix{<}l&m}Pe2;(v6;RAHh%djRc zl5BjCn*7q2dC05?<0DhMzm8mKg`dlaSO`z!cam38Uzmk|G!qN>X0sF2@BWTYdlTc@>G7OT zU>vs=dVIcA3ylAZ!}kPIj#CAGu^8JQ~8q*kasHku!N9 z7x#GGwNo1&{LuQg^VZ5bh&ZCXRXKcbbr>4O$ZFwj+X|i@;G~uxJJPYi*3+MtU*JdW z?P{*jV_8v2g$L-^5=gYuyKSp13*jvc^O~-(I98~pv=w3#Az^ABl#Iy*QzS`IYk}v3 zr4lhv{-c6|O|=;N39>X@Hp?$rBdF9bC@py&ed+_L!Z)#+mOegsbbQY>wam&Z(=d;; z4Ohv}pyDAilK48++jGssCBr~LCS9bVRcxK&V~G*G*L+l-&Wgz*3j|I6nd_JNd6`h9 zV7&P?#rpz1C(((4Y!`gV@yW?@t`hKJ=tZpYnOd|jw-x&P3BqJ&f0(%B9L7X5Wl`o8 z)aR|Qgx_17e_V0o@gLS-h;?`Q8mqcP#nnOrIwmf+dXOJZI(q#tCg@DC;^UAmn%7|V ze+o)?XTkFw;212*6<5svDvafspRx?4vY(5uu6?Kl*=qxAY`x4;P={gYS;bSJi z`4reL9s~1P50ni5uF0fx%p!jym`n8SOw`T(qTi#)u6;G8JhxpcJJ%jV$~WDQrvG}B7elJ6iMHL;ti&)1+| zQTn2o2fzf90n`G%VtFCVjstMhQeJ)~o=bFzh$)=Js=|=S20R7XJrSG&gUG(J^) zWkPMK8M#bS$72!?lZpkG&+~@9WU2jqQD<}`<)6Rx$VZe6b_STXmdV8mq|J~L` zJHgH24_dH$mA)!!=PH#=-!mVkQQ!WGnQpZ}xyuqg=K+u66dV#Zmp&+W@FW)wb5yo} z03HjQ(eQD}E0$2+({)uIg-g^%O>zv;;_$L`k$e}kQ)-j0Vrri?bEK+k>|`FOPRfj0 zjk%&@vNlzXl7}&TwGYzq#)dEEiJksD+9t-@=zZxVLcBHPnY&CuTGiB`TP@|ou#}ba z3T4~9$Hp_8Jjtd*>bP!k7<=kqS#LjaXZ}X-{Jxem>tV$p;T%N%T_Acxw8m2R+C#N@ zRswixiQT0Ml~99Jx3p1yEaZmXj_^S!0112{tD}rxa+!NawC{qKUVBcxpaX8kxxWaN z8n$JvWuOK=K+hjbE`%HV^4O<8@=Pgg?62_nG#rd0&gy{j>X1L^0=mAzc~+NrsNE*4 zH(ut(tEtdj+rWz}cdrV!_LKY7*>0@&hvkIb3zNm_%~xAL?jLcrBU_-3#Vp@J>W4UM z%(uTnDvQsu?6z#cI<_VslK(U#2gE-P;Xl)nD)Fnh2MVyNQi;EzI#rjJiT;V!i-Wo+Yq$wrL>(?@HmS)u+d*g3O>I;tfx_))e-GAy}yup}g=Rwa> zL6G=9@+$b3kuFLk%+1af^)qazD3$Yb`ET5?PI+Wl49zAl@~!)Pz-IYkBVr;rz814X zl*nhV3GM&VtbYcK?6FuQ+J&9We?%*VQc%UInx2tuD6)IU;5Xd4X1qZm+#QO6bQ$5xV<3Tau(9y6LfEY5>=$5Mv6tv2OF!XgRBONYS8g} z{?P<{%sChG`C8y`gD&n=m*Nu1c2 z%_-8M)^;1G0Dd6;2W*3E1brdeD56l=$Ax+xhjgH0Ovt0(nDRSbj*y?ApTq zxMnjb1lf{o2&@{bw(^-w73`rGsk|IO{WZEkmqa>V{{F!M;UOPGJ42_DrMfBTAt24} zIj#9O)*m0#&yFj1^M~YmZ0fg3ss6W+=5rz4p}m=WERzKzTV`e2O<7Md^=2*?)nv$s z4P1B3A4HL0b^~}%fsgPadYPwZG&hEomh_?BQhp8$C@z0|MweL4Fu@Am=->E6I;^@O zey0B6J6t%ZHu%1Hz*=_Fo!YtgxlFgz#F?pr;$30SX44j>c!}+QCPDhKPfT2AadDep z%XJ@i2MTY0y>nLvlnI#vKYe5E$WHsPoJI!^`eIfw4VSK{PxOnj&pmGU)IP5pAlV}k zct*HLx@%7P-zqYhKMI_DJlXMPC`{lCRmrDwFdFeTTZp*o>e9Y;iCvXHd;)$Xq8s%1 zbt8#rr@9KT{IgR^r{lS;j0Cs!UoF(}`e&b8aKo7W;cx5!+@`Kklr-hg9dgNEgT6B3 z-L&cXbox($E3DFdr3jox;jd|gd~zRbRSKf2>F^q)U%Vh`s#BdE`wW;q>~UWA-mprj znFKBg9Uu;RYk8I2N#9^|Pyat_7NP@L4SP|qU!%nxr2y>ow+<9D)H7DQ2IjWw2wS-G zsk*>>Xc>R5E1_G?o(5;yq{!&W6*WW#Z_YrlpY(E3jL>5X$zmwAq%0_5>T?w!~KFa(qqU<@Fx> zdD?Z#7ad$);iD%ayTv^n%|DU|{S9iX6Jlej(GAI<)#=yx#1V_|`vlATZ_-8z&QJ6# zU{*G`7xAO???HgtUf2%Ki~004Om+b6`cCu+NUtmVr{B1w2Kg?BQ-h16l(Bh=!5ruz z;6oo*$Y>fLqP}U!X3M5-Jyx@Yt&MlQ?<4<#r?>>qxYi~YCj*7P$^rpi4wk_Q3dSGU zDGzaT9X2@hlx{$qW=@DFO=%|UJCX~9cG1M(IE2e#vwjz?#<)TI{-&${PQrf! m;SAA-E-#YKDTJ`~4qcC4Eb`<_6b;6yA>^c#C2Pe^g8vUJ|2w(> literal 0 HcmV?d00001 diff --git a/images/Kaggle.png b/images/Kaggle.png new file mode 100644 index 0000000000000000000000000000000000000000..287e50b848df62f7e743ab09155aee0e1ebce3a2 GIT binary patch literal 9733 zcmX|HRahHLlnqc^iv}qk+_k~ot-+zVTY*4}TX88Kthl=tiWYZwhf*9$@nQuw{NL`r z`eQt6fPlrw4MrSdV zIe$Z1mD`e?MnAP-etTdKS%ok$TcySW&}m*lK}`gMktcnw=7uRXt@EQkLebLkYZ)UwHmiQ+{>^RCp0io;%ZEIv#oOLLE{7s6dYz|X7 zHkkKNHp$NZ5WGssPUe})z~wN!v7YA#Hk@0wmb6+T(=~$#S3?@P-$lSiB05VR8!Zzq ztX?h^#F!}vnki61Ja87!3ZOuCdDSImVBDKC46O7f`PT8Rh>!8$@v|t`C%+#7&2|Y6 z9(iCs=3}yjT$9@X!>Q=CuF?aW&kUGrf1y|Vt3@KoF-_^o_upCzH9mI;BGk3$}$wy(P_(=8RV z8pW3yWCjc~n+ZYf@&Op5o>8!md$+xAeP@aT%hpZZr}AnZS`GM^Hwi02N+_0GNdJ?M zLcv)XzG=RgUrHZ0|`R=Rf2wz`YLRbbs*>Y!@5AS+-C zEoRsHqGRGRF4wvlTTPux?nKSAs#W;T9KILWsE76w2S9@G7Lcnun!TVB1b*LYepfi$ zYV~Jm>p7JiI9d)9+Z2y6@L;__EC52aOlKTtgS7rbjs6kv0~qVFf-WJlrfa69Yr3Lr zI7ZOa^IS?Pc6BM0Q?ZuZ<-zmXUYGj~JNEY;=i#7LZdPT(k(V&>=32QR2etvQLZ$1q zWwnuuds}=}`#o>6qt5?aj^_p+gblHgFeX#`^((0nOS~#u{sg zv&JhBLO%VQF;Y<|3{1!uGB1l!KQKlqiXm^Rq7X4ygz-WOlB7JIAfObTu}_nawJM*L z1$;C?c_PB?(C$;0xH#@^A8zPsHhF4>8$;f~`m$l$l`-%;!kA9|ZWJ@un~A-T^Svj+ zn0~=`N=Ce2na(o@WJNjYHunX*bzgiRu&`?W0SSh%>&m?3M{A>Xvr`fs*T6ikg@l)d zLL`k9+$21%1HORM5LkM$qJ9wA06O8<_$OF>U&VW=n+^fWAlm%%wt|UDyxK>f6uw-} zjA?M_D}Y1&x}KT&jlA;2-G{1&pX%_!%**$XI?4X@alz>w7I-{;Tia5u^}U>wEWA6Xro?nkp$*I47dtC*zuDio-t4x$6QSmNe9Vlb;d|bauk{L>1!b7C z314%(k7bEj#o+zH{sNRPDFBDDO_=P|!Ihm0-V66G&a-u|cCCIqtD zXN1Ych9a`mB^$ebUmJ2k|Ly!r3s6UjsBr7kHoZWk>ty}=m*jYhQtux@VgRqYwYq}G zVV~ldTR(_XRR`HGhyOJ8sWmg`~1 zm30yNsEJSB8P6cS0rCIsbnYwWeK|4f-#@=f<9IF8(jND_+jH+;4>PhdN--IwLbKJ$ z7bY^EF@O4&;lNj968JsaM&aC;@rB+8od4wb;|{k!5D_1Hk*n1v+glp<_N&kR-`ni{EeZdtRHh14)Gp+Qb&W|h!!kuaGu1E14+F77ZzY>lf#pr z8m<$}fbX$Q!GY`3Ml|IwXrGa8()~-R{-L#52N-<65cTYKtjsj|T)bkwvr3@E9-`T6 z{QHosr#SB6-14jPM^aM12I|SE!J4YUqkDD2Yfcj(s!iWAq8j%vmgDQI&<7s2igeOB z{rg@INGr$n<|TQGnPjME{uSdUIIPC`kJ_Oi@MDW#9YNfNFV1FLdOX!155OAXV=8Hd zYIwulQD}8XeKZUA(L4T?NW@J%IX5Nu*-<{Tr`_`9X1m?B5Lil?7U8mMe+#VV_{oej zdwDt-b@NzT(^78GcYh|sZUnPua!9?}xsUbvTavR8)-PsG=ijHi_lQH5VuAqMS5rcb zAYNb7sE)_Dy3^<%xes+i{Jq{O0z<#*ud^>*A2PfYnK`GYq^@;!=kl&MU_Bfm+?0Zu z;<^{hKW%6Q&XZFoJNkAN(_+HiVGjl5+^RufEpOv2xBQye)5fPXQRu6tT)`^nv|60X z&q7D0yk;5cXrgqDoQ@yHx9nkN=5#ScI>tZ?7v6pnRi{HfWQIlYiwID%D5+BedMhWP zP8P>d3v`x5HzIqJ75fbyvv*W<`r{Vf(*dm-~H>6(F{222+- zKmD=e>FUJfG#_Kz-3SoUSp~cNc^7}Q4PP?xFGuJv?I^tw<^3{l9pf~|(2E`sV!PF4 zqQc%~U#~0&b;`|8kC5Ld@;?HA7EQF{mMSt{U|22z2fIu{zpbP(;|GU~(8-!|;@H?e zYS7*}Ir%wB=X&^r>qe*zZ$i9Xlt})5Q1aIPCrX=r&DT#KA3vv!V16M98-^GF*z%R+ z)5^PcP~@J&PMe=-OwxLbM)?zxVqGZTcQO@UBsxY=G1C{Gi8S7P!waI_?Nd}5OA_Sl zE>gcJ%sh$5?`<(Lm&Un*Sb*}|YwIU1!Gf|%See&B!#6Fu=SGX-MB7Fg!wONM$-&eo z@y>LbFI8IGqbbCH7^2YA)YD0k~3txXI?n`UV$;mIV>T#`5PmF-*uTSp4Y$jD*& zaZ&9^D_P21irbk7!A_7zTY3y5*ver-zUvJyD&e&%wF8!srcz{{FK@-D8_|Pk*f700 zGpBpV=m@3-wT|W=d70RFC{1_ z!&-^?&6ekK*~9=%1FuMlMaN}t75DsNi^>|K_g_waXAS{l^hPk5|0P$Y!AJ%q^Z=E} zXPXNAK$<)#g%0t&2sY~%So{PbtQC4Z3 zrLljc8SI~TzH!mQw*U@BK20d^MGEL*D>L<4qLF05`y*2C7r8bE#+2&lvUWV;Q#76S z3qSULr3i?6PWhtAsvgI^SXP6>Yod#(ln?J)Dj{=Tw-Ye!r5|OQS3W!LzILixF*mCy zBKb;_iz$;mEe)6&Ci;PhZA3)u4PR03lYcxdpXfQ(zX{kJY^_{{-&H zPL4n2Jf2uala3rB-LuJley8W7U55ng$JtqBm!gXCMzlR9?{|B+=YI#>-Dp#BDxol! z1(cVytGppWd&rf|9+&njSlSe3xpz!#w`$?FQY~6YH@2MC%%P9UxQxM}Mn);tzIhs} z8*23Fp~#r)=v)2ZhnOVbDjsq9bi6v`aZT@@^<&_pZQGennmH5ar%CDz<0mnS?R!=} zAfJA1*&j@_bZf283Z&i#&u1{==}}+5ZG>?QPNnp5yI<8_JEGI4^bjo757;F#(tQL$ zEAFaputE!Wz66b5Ij0Azw-MCSVxG53hZ<|PwmlEwf^p@ov#Ewd9Ga`?w%qbnuvE)| zeg_|QwKhQ|qH-0Tl(KCZbp3BB(q-nc24lQJffSOMV=3{^Q7Rh7$@wLJp%jL070si{ zY41I#blR^(jaWEet=AxS8p0fs1~^c#)z~ZKG(8*Bq-%a+Hu&m577I^@tF{QsX1^UP zvxdlmNDq7x@UbG*SQ45NE0YoQ%e*BLgmfY~j?hj8a$2kOZ$A{r51+)O`W#X{&*ha~ zB>ryBoz#4+w;T)w4$tXF4@qsyFza8@hErVeP1*wP-twh6Np9|@)4G;Zh@NU5wEh(E z3{G=(C&UQ1+{L{y?`Pc%9xpg`+m$-aWmz<$5J-@d7RF@H%K}(k69ucZK=aTec{&&j zdm|Y_EQtm0McC!al1(G#B9uOwebH-7JVCnW0KJ6c1#4p9^~q$fU2K@3k1PCxg}n5(kc2?lFu_L2?9iFY#msP8P4)Ue41V~Yz$pO zQ^9-UB52B8B@cgh?!vWOK4@0m0;mJ^Da>o4j^bDA*(k% z;vLbKFm}}hUVD(gFj>)kU}lBSdrH~ZxjE{jRmBnI%8{AEb|z-^O6V(SZfG{~B)9WL zif@r6&5ycj4!i#+OC@Vw;BfRF%gpa%>ekYQFdh~nP4ulzlsZ-S%%cCROH1oaggxZs z?|9R%SQ?KNAR7S{ z)Z$V8lsC#wleho&5CwDp4HIPywWvSf-H+%k(~1nWWF#q*&9Vgh+7k|ji%6|1f@4Ym z8d@=FzrCq7SIZhNq#|jy+2`hL3|AhwMPB1&-`uI8MnS9cm1xJwhYi@wVR8|*k5hqM zIyc!s(Fe*ey7j=CYEDfV;*KF2)Q)jpGTu#lOz-|I%KDHW_sUa0_F~n^2(0{Qf>ox6 z1v1+XPy1APg+Pnq<;|w%@7|yZ3l<}hxfj~Xs3VrCU~s=#pv4$2R@-8`<>dUz*^;Y~ zoyopV70K~o2Tiq&llvEVM^=e#Dw;v7&IM|qXCf|=>^`Sbh+9bO{0q!E+Y%De;dh#2n*tsd#) zfDin#rvtsLm8WtVktx*YjY9tNd-VTPd3TDy=j6>)kW7L$j4CP=FXcyTHa<=7wx<@g z1d0ow#`^8qW?x_ZP2@a516vX`26t2shq|56<2v{lY}YxIrPRoW|z&ktFXV zkyFusm4bHHN<8D~)}ljDme5e+5sK(7$M#){&6Xzjb38N-$u^Ogd<`vCFv57vmXX9s zNzPL~c}$9)t&4VIObQ~NUPN>tRG18EEy%Z!HdyP=9sZ&+<=w}0qJrWrRunt&;|BWm zw@vc>hsPRmCHw9ar8H7>f>iQrg&RYrGHT8L5<+y6mkeUvgz|`QOhX!WnACMJi^(D3 zgS8vVBz%vM@qDW=ARs6o?5(hjs7im+$&vk!<8>j5Ap3`DowRFZo9v+(9DdlVK+6dd zX_g%$ti-MC%~5fTJY*%Mpn{lTZ#vCYW~;*j|*5f6}9>+#gio%#QflnaJ8|d^^B$xFmet{N(`+|uuprt9#B?b(=kGW z`a+Eb4W>$E=7;nk^dWMgpagwPI22uaS%%$yuLCDG|6>x;O-qNVnp=KUhAR?n+FkP8 zMhnu%>?Mx$ooEU=MX;nYdkplp)WI6q2FcqxkIz`zkqbV|g5Loa>X6v=2*Jr1i~JFc z%mpg52Jm+d<+jFg!6FjyPJGp+3O?b%BBnY@vU_#Kosc_wr`TR93vS03+0%apHZ#Yl z%x-qAl%{13x{YJ?=kX)T(gaF2UzO2H2)DN;m305A5teV#p?D@=V+!FFNaJhJdGajrzmiHy!|Jwwz+b*r^Y!R9DvJ4yqoeGl^#~Ax- zs(jJjyDcWyG zh2mvMeTffM>dIVdbwwmMGEkg|fETPS8s?0dvGpk4vT^>(Dn~L>ZAHmIC7zVn&p|QV z(}yD4_4ngoG1P(jQ!e2s4Lo3m2Tkn#3Rl3Owb5`2mVbi~23A&hbWw4!1!(2k4rY+i zlhm(r;NAy=+Z@JcuB#zVpCgG8IoN1&6lD;iq`MA38$<9%0TxW~A}uI3IAtmwA4mYT z(bI=FUS)o`MU29^&UM$=w42)AYACgkE409mYI^OaQ`Aaoz{F+gmq+hvZfiv3!vaz3 z@7-EQf8!*ZBOT|3$Q)kEGXUb*NY*gNx&AWYDOQvzFfZp=Sx8r|$_X6`+D~u5N_({S zIjYn3?s>I?`?pslk{4D|B9X4?tCM@3;oO~*v`mvrAaD)U{P!WXIgvWqP@OxO_cdi_ zlQGYQbfiYNWdf9?Sjr^BVfvRmViiO_~ioM0ggy+vR7ZENy;;;v8%+}629(W@-f zA{^{B_#Q}}9LTCes|QH#Q#d~X<#VeAEzN+@As z#d_rBL&pL;xSZi17J7Kwtmzb%hun0sWRYdEF@v9#=rr8$BFX_0BM3+;zg z*tsP}V0vc4%>ChJI}*#@@I+Y&8q_pbm?n0^Ag(wp5`ZWo9Uf#17?jg{^A*GEx$AOF zUuewVZQ}VrCM_9`gIVi)hu2^HiKOs%Eti{SZwTVv;vfPuI?1v~Do0~bNb=ci{XN!&Qh75jgB55-7Q za#Bm$Kw4x|^Xi&L< zz6IN{VZRAFw@5vJ6x+*v)-xMnz4nt^?@x|NJvCQv$XAvKLG`un`tvU)G@Wgz#7Pk# z2DDqZKxx4>%p5v5C=1r=C>~Eb|6P?Qk@n$G3H%wMzVVY;?RzeA!&9*MoK4=${Ue6yiNe^mxWO-Mqj|LzG||S>j^`+FVR|UvK6g_CL65Kd14VGcwy(G*awGm!@D*T@4l!;ji% z#SbdH6teZ<^?%lZ{?FpB_QULFL}4ACCHRP8#MJ|4r|4|n&sOVA!mkMK-3X{8lq7M_ zpEmyxOf1fGe7b?(cWqQ!%On`w>fSIu8ve0VA;8x!{Kc#jsaGAchOJ0Hp8wYcDuSXZ zKmwU0Wp5JYn8b3Rk)FBqdkq;#oM{9VR`zvibE@N2^~HqagCS=lFAQL|p>H<4;w!=09xed-K0da4duoMQG%rI33W z&3)p;pnr>{9C$F2VQcV?X}p)U#g4cNxq#!_hd&CjZk1E}GfrlMPT$3X6u){p-g`Y? zJMFafxlu&8+HbJ98$W2@hLx=3iFI2dD&d~?%~Xl&wvsAtH>GZ$tdP6Y#R2)2bhSxx z7nrky5@*uCV|!|m9`GSVgBfP3WgO)trwJGN-^yBPuU?Ajh<|Jm$$$4AtAL0vg{hu_ zjI7gMUlA5^ktXwyJyjf8*)Eqzhq<3_1el2XXR-2W*@^SnB5$(jln6q@5_{4$wHNBb zZh~7w#3c*94*ML~<_%gA|42u+;VVhPb3@2~AQ80`Z=U$aNFhIdS(_!AfdHQpekReeHKq35kic1 zw$3-E(X2r_sA*OEk;}qmSD#VV8zIi6%8oM!buV6Q z%%LL`bC%c-uB#LXC=o`0Do{a71i|=IpwGrTrq*Zg>F=3(uo^;FjXyar-$v?Qrx959 zR|Hht^0O!Tvpvto5q4H5YWzvC%^_I$a6(Ee=vcz)3l!(*V7fTXq zl(_Pp;9kGRiN+h;;1=y+)NbA=w|Y|qIk!FsQEseWOP(EDGG-LT<$&RH?GNl!^`2 zR8`CaB(3X9Yb$Y!xVVZ!`2*#<1T;G9^<@XLIMyic6j$*}R5Cu^c@C!y9P+ysYU(D) zH&)(YmJd}%d2kBYc)u=3Pa3o>d+nZSbM?U#$Pk@pL1geViQR@8#VGm+ zX;^Uvem_gI`$yb`3NbRb`|u&v#CGvWY-=F=)+(kJ@yidt&*R%dW}QgHb{C z2X^!b6j}Ybtfnccz_TLUjgu?J*v{kOZ0C>W!7+JvCao|_bU&}xVDP|$a}#CnNzEtI zA*$p@zmB19G>zZp%nukx&fy$n_$Zq%@Zp)HkQfF!A7JGl;v3h*6v7-JR;&3nm(T9aFT_0`pyXLt+bx zCV>nyUSvtG z$Qc5h^uHsA-1-<;FR#P<=RLB5sAJ&Fd|iYgJCDK4>(BTvN+}mgFb}Hn8y#~?8pELO zbOS}szl8Gv7Blp32qyc&@16HEKLYDgq0aD{;F7Ztar9bFRcT9tm`2=5qx!EH&7Db1 zC0~KCu&-CEmvaW-;}ZX;>iPmcAl8DHQ%VZ`#ykT4^;YK(pf2On;U3lCTGtS;sJ>{5 zG2LlshQ`LqMa#@z)%V)OpZq;sw@pyT1pY=U2EqqEZuX7)M=;?qEo4gTcWofr#h*h>_vMLj zp3eOP!!<^4!nhapdoJXny%MEXzHC!t4#^|I;ih`ybLf0#orh4e+0FWq16V5qzKol; z6W!3*GF)j;>KrdWtlLPsVW5q~gM|Hwq~M`nBQ|v%mTKm_3(72veOXPWm3A%w@JJvD zopI7-M`^WgK2aWwO0E95I=XKE7?w>sR@JkxLO!KAXWF;-;{L#%6Mz*UQl-OLn$cG@ z(P*O#59md9ewa;6VoPcz#{LV?3A3#nURM&}2WGBrG}SyC z(t3V(+$SVG-$Kh}r=sj1^2#D5^$Y?2@Zy(rg*TD|39|o=OMXTbGSVE2{iph9p7ZnJYM*O>*!IVxIIrC8A$@j3_Y)HX7}OU02j=7f5es- z_{UjPmLQ9%vNs{oDswjUKduN>zrF7IZzyGT*0|61rTqc2Md31}QFXvhg1PgvcBokD zpG*b#98ULOQAsVOf3y)Q{!13dSo@;=WCnB=NEnK`w|yw*TEO*RLij0ne~M#sxf|QH zB#7xNU4h|Ojqj%LfPrjj=L%+V73w7GK*r>jC$V!`^mEU z0!H!-VF~%?xJy;O(^mO(E9gXw3u?#dV0(``N}BxOKWNbZ z48IT%@~K{z(8z|cv1p;M*tmAt4EKkyQxPmk#Vkj0nRE7+Rsm^}%I#cVxMg?~gLD?g wDv1j?z%02IR|=;D{6(-W(6T0V^8ozgB>sGXd!}UfGUNqNlvR_dmNNbPKO%E=DgXcg literal 0 HcmV?d00001 diff --git a/images/try live demo green.png b/images/try live demo green.png index 540ff5595feae0169b7af326cde9758ec347d2d7..2df56933bd319710f9c234229d1349c3612e853a 100644 GIT binary patch literal 14424 zcmZ{rbyU>Rx9CSu5D^tnkX9O{kuDV}DUoiZYv@i91f{!6>4t$BIs_Dkl-f6mu;iqZsk$nQWP5CU15H}4@3EE@>q#_iiUpybcv zf+Gmz2}JhID^>Tj?O8l;)s?2}{opP9N&eTbiLe5F@P)CS)h@jMhAP!e45`)MgwI`V z!{2YhNuZn1rZLY@4Z_X$l`c0VB?(Di-kf}O!WMW_hFezV^~v`2ITgfHw33(aEP&UHw(o#Z&?a~jWQ0OaYk?0qR8<1W)nPUt5dv~nxHwkAR zQd7=6AbU9UfQrafh>`xn*+xdwMOUVyFsGs*B&VY5AZ~AcsJdc(=;%w*5*Z;^=gPcg z_fA0TYVfm^P2$r`_JP>U0Qd@a#7KHKOM!5_Q+4yR{ARe|YnFJ*b@XirRvH7hK=o5$ z?y@IL&ziIBRJ~taujgQUh{m4uC>dFpNM`Lm@slv*srwUz>V>K?NpBahlkMv%Z2Z<6 z&27)Y4S$CNv5%GXTnPG==gZH^=raCgqM9oEk0-R}%NI7I;1p8-J`?&RXaQ@=8Ps&Y zRza<^lVnuGoTwiH(Kkbeg|U%SjC&2A1%`za6LF8rMH%O(;`*0{QWceMU8{NgsD?lK ztkP1(oTfXdnkn-V8{*6*XQXNJ^VV0b%{+Ru(mGBSbC;I&zrbF?1v$L-UCWo*t61_-N?G#T_ybtm@q4qxoB5ay|O|?S1_#*^M8` z@qGmQfiEB*Ghy_BQfc)f@avG|!b|bGXKZ$Y1RFj#`8YtkXECNsr+wP`uxl%p+*dMS zI6NdQT#hcYIP9s>F=#S^EvlP*0o!FP6bC2!dgCo#MguOrt2ZGxA3#qw;2Qz+ zb8h_NuNIV2&x!>`XQKpL%cTkHUlkr5%%Xu=X zClwwGGJ2*v)}4s?3#tA7)TCKFNUKNnWFEiiUB9ISF*<`?ig^1Zy=7{wREjY*%GAbM z`^}kLrqh1LoEwmZhdo>R0SWzm7B5L|2`_A2vPTwiVA&lJi~dWQ{ZeUQMl-&8Y`F1q@eChxv)qMj9v ze_sSxmVV!>u3u<`dirBYOugr<*-Jw;v*T_vhgF^g+THAk@K&?-ZkR-(sF!;?HpR2h zU&JF2$R1u#U|_EuO;`HfRxZn>m}z2Wc+G*c}YBEDq-S&(YtY znIIvtsb6!$9lL@d;4D2iQ?Vfsp|AJuJ@tn2^%V9ca9a*6?+hsH`9UKJJC^0}|P_R@--uAjJP5tyOghnK^<#??@>@OYI|BoL{{x1FQ@$8q>>K9E6?fro``i4|80yQUJ8r*M*N(*0eOiF<@&o8WZ>X8Ul7=S)vR(MR#D0- z>^!OD=6j?osp6A?$A=4nB*_?QE?>3WZ8^A-Q#toR4Q&4z_uu}Rox1C3GEj8us0j2T z%&K%RgX_9~i|cx`To`GGJzak4d(~lp&!-FnTPD!%XxBgSxrOY-EQg%mnW(r{Yc z#c>3xGmoa&^v8lgvb%(G%EPV}m851TK-MjnGWw8 zA+AAR%SpmlyyVTki$Zy-3H3ypbl1ZKxA_7)X#a~Js}RUXTV`N~*N-o(?NSornx;xJ z`>Xx8dk#K@`ZoSe=TDv1_DVK%f&Ih+pScLI6%^!)Gzd^u^A{d@h!T=;OJ8OifS6 zT^kZdAPk+#j7<+oCIo?@mDE0@qqJDFR|S0#66tqdbdk^C)( zIG8;?BkY;))O8!in2O_LMs#EFKVig(8s)$5$tFn77*p@2NwNDjr@I$tbs>4@A_grS zV1W7b?J^1b#}P+hEI)VRmFx;w$#;8BZvvwcV?u^}(L7TqhHq0&J?*=Ru_pV>4?%x* zCdd~tQ14?dZ|)64X&2-dvdt?*NCjAhdcvL}q!L_CbJvQ*SS+Nq{&~EYlp(Knhvz0bsB!s+SAD&<7(+2|;ycNyr`9`n)hfu+t8 zjt-;OLJmhRI~aPc?(yH}9t5*&&0CHrStg9cn#P$>*pH{91saj!wAl zXKCxn+*i?Y+fpH}5KN=?5hhH?cRO3`HCWU5(){xZyX0V1Z>Xc;U~`ai(8f}IAN<^lUQRa8afPGd z;RFe(ZMxjQHl;zE@xy8NY@|`ooDx0Xi39nB%*VvGbG~tu zMmUBH`)`N;N#`HLNZ2oX1|7ji6A2If`18LPibZ3F&KYF#9I4sgm=x;|T?-evcMBA$ zI!ga??D}9ht*64bY)zV<0XC&Q~Nh+29~B&Rvm>?U#H^r`{5ZY>HD>}Ht*yq^6Bo%v6U^_jtS7pS%y0cWR-dJyh%saPSoTy)o?B~; zU;KFm#>?$G1B&#K9gG9nFYvs=M@LH}0yB#NLQ%vM ziYop$&yQsc$?~vd7rOqN2#l}I<%^NrhhRC9SLF?b-T1&RLFyIPY+M;Dd^A*q;`l(r zoWcE*Tk?j}YI;Ip|F#?apY_dE{`Weu%Gz&v{6Nt6;2BRppb~ncQAO1^o^cDPZkMnt zUyC&B6g3T#+_s46qt9jtl2W(uS5OZ!Z}azP;@QFBWn*dRLDwW`eXJCgvWuj(j7GMFDn$hnGc1 zly^C%sx3wBjs2JU+=1J?;&$W&pFW!PS!f#yYwCzS`R#Ta(pqENPx;MLx?{cB-2L5` zdOn{Da*Ba3R5#hJrG~Mlqw7@!De=M4Mou1Em%auKTS=I5fj$_>6kysi(wDq%J-4&>p#y;$b$07k9GaVoFG82P7xSfGAeyT@;*P zFL=j{egdvf>;ylMmsNT#wt~h&_W#%dCZyFXH8z6%fA&XERvo{{)-og)qffoEFzt8X zYO3^4L(=GrKaU_@5Kb)ch)T4LuqD%t-uGN7^i}@)jFnl_Ycmg_t~T?o6rqt=z#rmj zp>jFPMaQ6_mn`&BM^#AldwvK3p7^F(kq$Y$dtdm3x0&oi(ZOuSV zA*66VXB?vs@`PMq`_<=pFc}|J$CuDjp+VDW^4k|16P4WOX!`O~j;7$E!mnCI2F~A6 zkLC`0t+kQ96)#3Dk7+a#X^wjiC?clnn@hFgUyKw60L%12K? z!-`Q<{NmZRiFs&RSNZhWSn_<=XTz(HmZLQUz4BXz|E=sH;#D8!5e6 zp)l-lG_Hg75h=NY`o4nKl(3>rW*oJuAKS!wZ0wYpCHBZni8n8wn4ToAu>RT9TLiqI zSsPePjV5CRm4(klm^*Tzc9wQ3xYo}{fIYQxSX57pRPb8k$Ll)X zzW5zwzh$cv`U^nY1aR`V;9hN*s_^+?XMI zPbmggVkoDRs*=jX*IeW2-m1#W^7OD#A7b8xpbwU`Wa0N`b~y3w?)|dkzj`fOba)CdHbhR4v?b}AZ0ywP z*$`(=cC)mG7#{n z2f{+^df_2*DNs)V{(q*0i^>aOYgpwv}AKATEvfoYcs8J4R zzKU1zVa?x}URzFLKWsBIJ z@3wCe_{=|lk6WNG)TH#sq}`H@S>TUSX9Sy+Z|165xO&UFpa0^#f=AA?8xpU>kHlPa z!R1u#5OzplZ7sO!lub0^N6;kS;E>$1##4UUOL0s_uW67JR~qyyr-13iZ)Hg3uCpl6 ze-LZhnN`1Wuhz*ST3P@WltoR}urGDx5|j~Gx@?k0{HXkjbNT)8yK8(AUr~>nhhr0d z%2mU-6eMFMCOXm`d4-8mdfD6N2ItjBQNW=7_L1JOjNbQUhLPCZsD#|thl&FW9*l6 zOV#f0Y?Rzc%L$eFx_K1hZ(r*M3L%!=h?wQfFo9?BjVc-jEG`GSx}9hya@>Y)zlB>R zE*#22JUvEiO3jhS!6E&z_`TS5#3L7tmbNWHX&s~yjoSDeb2DRQ0)LyGjB2L5roN-S z$Y`9kti;>rl-dl`P11ID8ARTGsqXx*AtT-4e!Kj#KR=Vxm>HKmGkP zST%YSr&|+5Q~UbU;-?=x)exH(WaI8mtGCcCubn@SQ5Z2`BXOme3Xi@#4qmcdUPc!;;=}!XS0?xp z5u(I9hz+P<(G+AT4J?=z|~g7mjlnw!4+pr&OPoyA!d%EQa+jV@2;%--GhIW z!<+QtUu39vqKZcb&!26#AjQ@x9+gwY?knD%O8l|2n=!F8y#Ib^%#YePm(Q%HL3#Tc z1jq`m*Tp4yYvAN3m$AEz(e2DO(R^{KAN%qnctS^%0&f;lL!m;VIv%0-+AuOq*k~GUP2~GaVm$$ISDhivsnc*NSCy#=|3VIjq~??QQ?wsq12W?ERIG zr<=O2wObzELOE%8dXv7~&1=lU#d=jH{?VgyFQdfR+#_A8!eVF7#$a7e-*mTcu4|?F zm*~&l@UPerX5QVKU9;%4un(>rFrq2nF}oNcRgH~UM`D_|Ee(E?yDFF2&~h85>C1p> zk;qP%V2aHXe~q?mYCj>@EiM%L1;=v(uedVlh&~YLMDe3r-WzB}tX~NV_UDrRjfvCc zeIkThRrqW(UWF5BTHQG{)*r-+m&-(MJ#tb+1TRH(B=3GuwJ#tip4TCWkR6M#GdFYU zIo{y|sXL^IhKOYi|I@ z@J-xXaPSXSe3_&>vfL@dJ#8yt8vVhobDPSP)Z}EMjufjdt1EJDcH`44p6cXaPo@uo z`(>rf&YqA8Z$7dImyV0-+wj~MpXTrs+2hNUxsAG_RdaVST-n7_U6(f-<;P z!_)m0RJc(UP(2B3Ks=v*=5Jbht?apU=46DAHWoe(*ajA|> zyFVvf*|`&*QN{1^y1C&!{`=-)shbM!k-#ccQ2)46+kfCItr%7Q&{nOg*=Kl1uYrx5 zcr&E>M*UMstf-@#`Yroxi4DgqMhC5KG}hnR=02Og8+hnq?%Lcw?)Aj&ySmWJ-A_+< zpYFO0oUE`LbS*E7m>*70nL~%_e#D%{pC)ea-P(wLW>+@s!$ikO)4ToB6_UnQaHv=^qxWWEF`QamRh{& zD@zgGo{Vsh>sw|QGgtUls5sTRRG;Q4KO6Ig&uK~H;N17`h5DDS*BVwMiGHKD3tPwC zI(7#psSJTgg|DKr-;}M)to+(*$bJFu59A0+#AosMqC~WPuDYa-Lua4sH9zQ@Egf-r z``d$(p6UCk;+EQdPt6x+8vzEziFax&NMGwLfAZ6M>(|hfbnrz6XAgC(-(}t7)#c^! zTCCXHtSPpl@w!AG+lQM+i5+LUND*~4H;lw3I@u!7Osru3Hb?^0M0XK+D%++VmhNDG za5ff8?SdC6es;2JwO?6^`kb9Q<4ff>lM!!M!;hTuI?GnLw*Q-_>=KyOfbd)3%#!mm z7`D!G&HfO062X;HW2cjMyj5S{=Vs$)fS+>mI`s&qu%S7LwaTEy#>~we*ospg6*|$D zojRg|J63yqAdV7MB?uq&YpgB%{!X;fJojlTdq8`x#>>bNNUXNu`LvL{4(%_J4TrOZ zW#4w3EODnZ3iy0KYv0WJP3ZjJw$5B!H8rQ~J~vnbwZrM_C!@RD3Vcx&1*eZO2GZQ) z+$RJdPlM1Er;UA?Why+QnNV~LkcMSxLE@ANdC=GgO`jOvziZr@A<-xj(iKTm$P z5}G^>C&?A1p1LeNIh&y-yXNa(_`u7V07P^>s$zIcKgA!I?va9E_3LIV4GJL#a-c=|UeQ2}Z4y7fIR?D%3Z+w}hKQgSL2BbW!*) z`GJgK)B0Wd=d@kR*c!536q#A#lP}Fie9f31p7m5NvC=gwMI41H?oC;H#Wo?-r;Oi) z@<|OAoFsf~KI#yg)qgH%X8JuWzp03b-Si+hpGRQD3B;muyu6=nrx6x`c;>;n54f6- zelmL9hG)Z>rkde5_G1a#YdQK;6LD9b%1>>Lmgt-p9sb&vvDrsi$fXD;O%Z7vGIk;Q?$wH7_VIxtO)KRci&JlBT?j8XCL9v-=b4m8 zhYHBIj_x91mU9J6s|xRnZ8XEKN;Hx$_E44EL5ZdIBR$)eWG#4D8+ zwhtrU^vQ!O3nn3!wR`)q61#pzuX^0h-`WS$k?oYnW$s4OE?-GtHqi~gpYoY~j0oG# z{qDT8w;f(}5O@TeY`Fb1SjuV%<^8HHyLW74sec*UE0%zsvf0lu)7E`I-?&pN!#6XK z^|$`85qckvVm5!;102h<7HW1P58i9{z1#?d9Ez|QfisTC&>y=bPOERDkaQWZ%)mTAHl%uq9 zwCzJZKDK(uG8i0{1MYt#>1l+^tADQd2#{7!Wj`qYA`QDC5(hPBp{?8V;oDFz-!j6Z zTomFFRg(_S?q_5)nMzOR6Jb=Ks#`Xnn^1*HY2QzedE`hacf3edbwvBt@^f6p;e6F0 z>kD&~3Nk6DFwB!&ttFU{kA4M$^!$Rr=-PQtR-5|WsQ zB*>J9oR9EcotWVY6=CftM6c;aF+e7~ymK^G+ey<3f7?E^{u&ALu8I2i_@Tc@z#ZIPrF2DAx z3kX5Q67jfrF{CGRE+};>YD9Bciq9W^9YZf8327@esT<@5rl*&DbJXP-? zd?wU!^7>+1Oz283)k+O2SJgLvyk7eaJM0NC)1tJ$COolU@~e;Kv^6h%`S%FKfFh%2cAaH7FRF*(8-~i zc0RvtD*R_V%nth^OcJL3MT9i`?mZ~yH1eju+_S-+IB(M>%V6ENIakqI-AQr4w`LQ% z@oMMbTDwB|a;H^0(@sMpIwLhaKSa&Uq@=I;eiIUxo7z%dw|>V2n}u?-_vbt~f~ukx z7)Q}7(l`e_DYZl;RkTYpzB3$45B&I-iJ;zzHQiI1G}6DoAGHWNuO54{pLZio2QOu% zVoUO<1}CE(Zh0x`a9g@1`QKfJS`4@Nuh0HGE)l9ykK8Tnr{RB85D<}jGJ3S!%;zOw za^N3#Pk_H-Qits|y9W_gx)xg-7y(a=19U7op6GkIV$U$OvD}~mp1tSKFsN+g7vdM4?P-!oMTaqR|p-s76DyIXFN_c{V=V)E=jkK9k>gWX>~6t&c#F^yM+Iyj%1S*jTOh&LfR8M+y170uk^U6lTAo`=BomSpfmwV>jRz3=^&CqTE zcyhW~s1UdVEDlVn5yjC^317)fEdn35&_7c=-?!oVdBdWn%LrCA|6qs8i5WxBU3D_0 zH8yS%6?MTumC+dsH#Efjc$;$KT$;(l&V+d(D1DTnKA<|66J=V6z^``wyleH=rL)qb z{&sbXXQ6%QRC!g7@){D?$niA|bX!G-{-{m}ryDH0-^p4##mJW3ys8dm9B#BD`{foZ z??Fm6rCTIHzvr(NjpOKK(kw6yGPCLPLB3M=4* z3STK&)nESI^Hj6V&u3$#pV^tJ$T-V8NSbwIIp{h7iP@l*-PR0QRQ;TUbLSDV*B&qS zBVh?ysgvO_jWgC+l89B@&yeGtncOuLlaG|AX}0g+HCAa%LzNhdXhFE?@NkQHj0Qrj ztsfID3XMgRUrk9xX{B>F43pIT4p-``5T0yJc9DRB^EY2OiVJ%3i?!RPPk@3)F#HLz z@=3;+0sQGR0oB>)^zvJ*bhGnr-^m}8UZ=nUr`~e(MM$<*rahVv6rT zKP!AnP36+Cx~9P6oV{YJOVV_82?7k)5f}c$_^vJBurC_+xXaWeE#Ck5o9!k~_it#t zJtv2-LDLHq`SHktoIHQhoc13H4gq8>FK`e-D6B?we$Lk&&tXp8lt9w#n$lt`f)KZe z-NEfe^{KPWo7?CQBjyQ9X0GQiXFXlEshW1?o|%7YrJa=LPJ!X@alPOhZQ`L@U~pVx z50%^0yOWb>W}7(}Bzh3_5)*W(ZgBK~vOv)L9jWOkE{`hr{PQwxzxbNk-}c9bEwVx3 z=IE7=g;VBDBhrOQGC z6Y*;%(kq|MM}oXMD*R%z8m^?ru+6m1?6R`;nU^K*d1vViCq!x;%p#2qgsrv>ciU{n z;%L-B=0mRLO(5zvm-r^XY-!2`aiUj}?Erb{n0%eyY&?(pX&b{gBhR;G?a~&098-lr zbWLjNJsdUTJRSG$CXVaGLD8q+9zVr$I0E00aDM)~=jWSWpONn+%s!NN1XF9sc}!s9 zb{sNUi5cRXmB!o2vz*6@Q1lF*EjPZ(>#RTn(V>JfqsBSf8ym;psfr2}TB6EO%z5bf zb~CH_7JQ;AO55<8ym)W}`DO)li6{SKb?%Iv0ck$><0{$;=YBJD+eaAa;8)6bPC6}a zbNc}k7cDEl^xTY=mVgAZ4|WlC`l*l##Ge9JqQp>F^Yr4H9-AL=dls|fTU#=ExqKA; z(ZQ!BBkW-(s1U7jowT^9Lf7%QLXck@k!HZ{Zzw?i{L@{1Lp$m*P3E`w9D6gJ8ntxQ z0inv$X4)zdyROHpNAj=S-g>NK+~nyTa}5+$26{4Ae{#T)sYtl$5RN40rdIh>)-;N@ zJaJJogQsm5Uz+=^#7ZZ1#9WNLhoQ6aVRw2$9JEG{zM}h7=_*a6c}7hoF9byBcf-RZ zsDAUvroM|B=NZf$rOM$y{d!rfmqCv`@nNIOEQLqfNa~G^vYCmEva%1aKSAChxbVXF zU#U>0g_%C7xXJi8IWgXE3~S5ryQ?|$4a>*B<8%~oBAkdW$(=+mF%>^@&pL9-Sff*= z9)wIqEKQx^+krIJVtOj)gWBQ2*+w?U!L#8-yGGc?Ca_imR)znlf=v=psS^OOk3s)>@G6HCD(_v7N!y(y>G_turN}xK z6C@rpX%RQ57|iFtPZV|yU6a z+xq?s-^r2ExM^`5aZt~09X#*4`*XU6Z8Y!H3*9z2$ECP#;O1R0D#k;H7k$u$B*b$e z+6h|)dcogy)nbiCqbO)7Ba7%dWHh%!QgJOeH~E@#)f1==Jau}OcKoh)h@ltH46B`Z zCbj&buV1fv-dS(!UAFa^Lztopv8^e~jMv!(k$!5vs+ShS;?mB^^Za{&(R-UR zs%IbiHLyp(^?V=4@?T&I9qsrC;}OS#WZKc*yiqjr?h28_^6yHdS|zAs4aUyp3g#M zTl=xV=*1}$8duSjz3mu)LnW>|{eC@io?pr0Judcdd#oMC{Q||3f zJ(*_#{m^tW!s7RB*{UY*3J@#lEnW>)Z{fSMj@agG*!^d~F<6_;;5}dI_d9BAbuB=^ zt~+=+d?iHrKz=l{swC5MEeGIW=6$U~yLEvVH#WQXz=u65q~Tf+Z>qn7yi)&V8sfj8 z1HdKp$!s))98iNgVx}8ihHI5%6YOuh|8hTl@#i5FU>yq-F-`OVlb$@Y$9CX0-|`8Q zzuzP9nv~UQFxaS5w;Qq0F#NBj*6~3Q@26clbo84Vt)72Fnges21-wKFe_`PaE(78FiiY&tML=x0BvK?Fhc?84E zxnn50Fm3g{VQSih`Rqte3H)DOfYB(pR+^;9`78AHbQ`?k0O-L-9abu>ZU%N-MDfr; zLgd%orq399!am%D8Tr2KN;DBWwN}`x7ynT^x~uAj=nv-$Zql8(cG=-ztZ8(U zvfOdP4Lzl;j39o!47MQxz^v`9@4optnA5L@x)Wv3e`?rRSa?MsAOZCOSj~jY(l9kHChU%Dt22fsh3WRDfSfcV zci1<7sP_94l!w+|4qVUYHy|Hd8MI`$qoakhq`kZvz6n>C!3C`jQWBe40Dtyj9UtIF zNxD*RdLvv^c9R}HqSQ8DP9UFZXEed4!7#4>hH>K;Em_xkMP4J}gleZyHow_jgFoTp z0O#VQLyBdTkXl79M!qQpjDT@|sgDOiq!eID-IyX@Ybe$ESlP(9!}-OEG(0B)ZT3K> zZS~LsutCmFBuJ6DJJ#Jm+jQd-YA!O;9Psg=0L@8-$^1j;BG=&3-_!eaug{rMm8e>J zH|;QrCSm-?%dg((PhX-JnyP84-RwB{X`$5A6E}Gr-C6d=4pTz@AZ@pZGc4RKQ#JDD+WrXP41A-oKXcvRK831+Vcv|{~702 z=6~2&$kRy)?k^^URX#?;xA?dKjeU%PKwlyhHiNgu z{1v=4Vb`d?Ug~UyRN$FUJ&o3V=N}7Xhvjd3bmr1iYXBXu3}{(2hKqHta$lpwOlh5j z6=P`*rhg2tbp{1$xScW*d}%dZtYap_i7wKHjnU z4onoqNPlLFL?n6a@+nN%AXp`bRPJ!>ldm!zUHVx(*U9tB9i5Z2B$?M(V8LvcYMVi3JhK>RY8G zPd%TlNE*>2>cxZ@5(o)(Iuv<1uu^G?EXtJPD(w!ODiGeS7$ozSOTJ^on8wOT4WIOb zRmhVCEnw3rV024&9KZj;KbQJRdK9TB^knJxG85Me80~Mjs21=+4`I9gf8WQI z#62#ATB5Y($A|_erpO0J=E%pU#z;o;rUI~hu5Wxy#=s>%UcG{T5dDWd1q{N_;(-Pg+L(qa%4u7y#iowD)Y~Yp zub3SiwdNQdJ@&}2ytd+apRYC5h?o85!H0Jm6-D;%@aXBMOjv|*uU9?09k99uOOxcx z+#8cwD#OXY3myowIEk&EPn{(RyDB&PLd$tGVk`K}qRaWrpk@3Soxa+Ym5iE|{4Ap> zwxRX{Q?yfV75TnH6USE^2IdTtLc$D_eXuVu7Z^4y$4NpTXjOVE$H$;qLc-wP<)^7u ze}6p&|5v(ck}bWz0s>OL;p19=#ru}6a5oq!cQ-gs88^H@5ihtvG5o7w5+#w7?o}em zg7_VTSaxO_Q}M1Mtl@&%7D?;zvzoIqyPEUFM8Vrm!CWRlomH@Z>JDUnh&{47HD56D zgH*NJo|OO3KfzyKv(TV+QlL-7zokaa=)Q2*`jH6J#KGn+x6(J zqb!!VvKJ;qoHroPOi)1~9P!j~#oku0Rw^uUb&O>1XQ1kE{7S=VtdTVPIM8zc}&cuB{2UoITSSA^fRzlII&D4tR6N$yt~msjwl?w!)|fVFtGh)`oi z8gFt0F63oO5o3V#utB1bNzcVFJ)M3^>@{sx^T8biZCL^ObR@TvaPH7hY5~9G0TDfD zz3_b`Q-Ymrm+wyIjaIE5CCUq;tahF|2xj5_H@y3yL+AFMxtsI(-OMnrV%g1k{ch>t3z`mq8I2QA&t3pu)}FNLxO zy>BV=BPpNMk z1pP@0){3Rmec5pf^61eUvU?N_j%vpNq&y@zb}P}sflceT;w>BMs^0E>LI2UYr34ld z+|)(=xb61@NgPk>%!gabICTatyRoY(+f}YIGwa_lbsjZ4se4H`*9%4^Hwt?H(tGjs z`OPdp&E-I{i@jeJ=dPo1p)3|RArLBdl0Sp4GvXGVboLLQc`@OeQ27mgDc5W-a9=r% z#i<;QRb;}{!+Dd*WMb>>uv=!nlCk71>u>P2X+l!MVB%L+xbIbeLTB0mZUO|-gdO`g zM{-o;gI>6MH3g1K#aPktx}E(&D6y%P}Y^?#SO?uT=w7+mbycS+3qa$mNYi3 zdI>Bs2Qlrk%LbNkn~U(w-eVgs0cr?jUgoENYY-i!(8k!wva%0nM4xrj?cSpimQ|4t zCavmb$yg8w;gDI`we#AhsT16B6z9D$j{dyO_U6~K<%K?414DfWLJ433hmX7dY<}aq zdOyA#E`=S)hojz#1D%gH4Wku3L>Xt~hZ@()X0J34bHCJPDA0XKMLavd!}4nfE6YZv`f$F+6v_+KtBkZ- zEg^D>J3Hr#_n{H+^;wH*Php#Mq5Jp!Z+4)I`}W>~_SLkJ3f$;+uP8pLv9H(LEsgG% zI1q@~t3=e*ROA6IEmdMg``6+#r9JB8Q|k=~AcxU2W#l4VYuCauIl6oTda-lr9lLAa`!X?tn?#j%zL4M`c8LaXWBM z9QER{brEtrvbhQQ2zjsb?CohMdD~pouQ9O_73^O0@ygQgS}l$_sFM}*JD}*TM_s1h zu=u&?e24y|7d;6s^zur#liNMPYTz+3quj!8lv$KqbbOMdp9PvKd&3eAMTNX5b zumgcP&GQn!VhC#x*^3pu9oA~)FUp|Nim9T*X8!Xp6F*<}6sHg?>#_h5ddQDpRa4nJ+0_HE@H z8_wyyHt{=PQXpZG{eREWry3SQV|pKP34Fop_^z}r%>u>ic4wTg*F0cUQlyOg7KWonMs(fQ9wZ zFovB{vE+??th6dd&%w@2_i{aFvX=H-1 z@J}x0KlsQ?!R&Q+a!+X^4+E<}*+lmkZ+*+(uGSP$`Ef9tyQ1&~#;$np`ON>E zBs&*pReSds_~qqJBs)udZ#Vx7UI(#%t}_D&epnG!@mz&6Wgj*5=nU6XeWZ;igYs?p9TuK2Hi zI`|gjV*H-pGywt|W=f(BCSuArfgCjZiaXgv)Rn4rtLg9IULW22pTJptG?L%$^|1TE z=1`C?-HRv_QIyxVmq6vsRi5@r5m^|bU}<}El{oI^>Xa#t{e+GYU-AYX*b|@09!K3<>kNcFbrAD) zY_I<}pf}|mU2i)l4EZ_1z=6T$9hkKHZj4y+WX{L_!ZZSLP{I%8ZU}Uh~wZ8^bR8=;U47|Czk#SBoT2{$8 zh$*`d$7f;?8l)h&-CJKdlmr&~M_PC-&rbsvtX01@WMm=&aRv$@*DRE3B|oS94lt*h zG>qU7Dj@hz`dYt({1X4le~lIWj?A;55E#>>X{;$Lf9^ku4VvSClQQu<$|h51hlmu%=UvbvUoliSO^20QkyD3d8Vm4 zbAPSeISb=JZlGo+2AF4>{z@zl{4L`IwW!}k)7akbRmlx5$lkID&4Pgr=I@}w;Pkea znu&f+1|#39|N9mfHAA^i%!jqw9=oVLfKlJ!i{Y0#Mh9AO6spL$*m;oV#Xo@G5`4lJ6FVCZ<3-N|qD=9gz^SQvojs}}z#x=YMY z8~S>1|I^)7AMq6Ipo61d^5C&w*mW_sz8#_XbYk#~copm47W7|RxL_JGR=nox-1xxW z2$@GFPwacmPmjG33?jpfKc$CR*MXmtujI1Dzedc~;yEOju7{n=7%_L1{+f^1HBR#a zM?Bi&lxolw$Z8spa^b_jbX74RbLdgX1`k<>k?>aoaM2(}*dOF;Z zzOvs{sCROiJck>%^cJyY{`g*b9(uuKVz(AX%zGfqMUb%ZTF03X~r=%uRJl75l zSJ1w!x%N#n))+am{ENN}Vn8>0Uk@7M6}5+K5WRdU7aJldO22`ZuTU@SaC4W(ec&6~ zHLXC6#3Mi{H^N6kcg@T3mx!`3q=_MqQUebmlZXr)?EZND*KCC%X3nVb;Bl3aI84bZ zxkk|YmN^~dt&nxw@J+BE)aWNr?20cg-p=m#k@%9r#>}XA?g@c>?@4%GF04l6N3yHH zEzv={t{aYry9x~QdqKAOyvT&Q%KWVhRd>Cp8N!j-ksMZ>~Q_Yw$f4$ONfegLb1c>-yM#?>#Yn% zK0L&S%;Tcqypo-7c4szh3Wf78t2QE2ONS2w*iCsXpJa6E_GO=+e{`9N{|5ima!$q#N9rb^!#WmwY~gb|3ze$ZrxHojNcgg zf-E!*(IWYCJ9~WlvaWVmFn%C2?8Qj7YZqazZ}My(?kCGHM=I)->EY0*KRi&a*CFz_ z;aU=+WVtc2!?w`MH&uK1&(w zu8wT=OqSQ>tVDa{ZhjQ<%ADNlTR~O>abP);BiY}fF|-5M7bo>GWpYy1Q9`5oi{y>2 zRS_mV5wQiOHrBY6^@7V(lXiR+7I@t|2Kju@6p`yJSCddpaS9P7{2=+FYR-Bu%4I1Q zWKEbBhr^Y~wc}B}ga$Obw?NL5igh?t&*-3oe0Yv!1_|y}>sNm#k;Z6nV0v-s-LpTJ zwrag)8(cP~LXcO_I6P&&qrriXuC1MWZtDLjSX6vF7rk;wx=#ud!@; z7DGgo`{{hW{w$8T_3=o@_Xn_{j|=EZhSi*WSk_3;pPS`1f4)+s>_&IIORm{Wn__y%JD6ZBK&X4!!u8NSjAqv^8ETe zMd-CC-==(dW3l>WzDfAmRL+bYU2y4~wPVsHyKh(xAFPAP51Dnvs+bWVl`FTmj^6eo zfxgoy_sDU}@+fx8l595>t<+fLftomEU!4a8%6A!o-D4fsvYSIq8>S&Ukk_^Lc^e^QeScykEYrlOm!NzrDQ07`{!N@@#l_$as3W%oM(8+}&h zm1Q5Wk6o8v08JYD>y$Gu8<#r4NWtQmFj(~L8in^)m%Hyb58yv)_cj8VY)W`EYV^i51zQ&)$8-t zPGe+4n&!!l0@GmkeMW;b_1}1d>prlG$rbC(^2+J{3}UlPHno{*FsJl9hju zMMgh==FJQ~7kN~{;Leib;1^vt?hXOTFv+l09RDmTmt~F*ja+cHYhqS6D6VJ!u_k+~Ey}bu*GmZSU21kkCUkm=-&^`TPADKrvoY6}pu<(JK&qjX} zl4bwIK-9{HJ6?jBZ%yt_=rE1_a!_IB@`lS(Hd|`r`m@z_Zy(RSacg_ZD3}uRYFx`) z&aEdbWav_x*oWyBRpdOArt>;?G!-&S_GD?~sXH26qfgw|qEF%$BNmJK`9?HxQ;E>2 zb(E}aOONR3(JYc&JQ+9hWpqv6>(`teR8c7rCb{BMXBEiP+IZBOOSM@U96H+3(K^PT zy_>xk{<$h~C@?qUuhX;#jrGom=F^qR-@o%(OH35`yaG*>`n(yiX8uUFj*KvSKP}7^ zD(JDO9LZobN-iZSE$6c2L569NaKmZ~YI>72HgZ*t*Yo;j-ww|@@-6qJ6MN<5v~8tD z?mjfXFT!{+nJ96&^wnMWq_Ez4OaJ7aq2P8D$I`1M=xO`H#MDI`zemo?M5$N3l_hPHTeMRV7;wG*j!;WQig&iLHbd zEcq^HGV`!zF_~C(nd^HQyH);F-DrSM6C;f%q+1Iw8+J2)CkS&C<1(W9JRMCwT!G-*%Fpg4 z60;M&&_a@CxlANwu$QK0{DHwbAGMBA6Gi=`r4!Tk_C6P%5%NjpKTOLgmE6kX8e@*# z%H|p}E7ofqGh-%zIz{1FlPY?yD%Pwu+GTCLnYH#cjye0WzlOcTG!{@dhDg^BoK>JL zSn{gV@EOm+VZm+blGfh>&e=(KbTnJ3VM^1;#fMuL>X2QbIPvwn$yROKEb^}W#yi|G zsj~%(ud$a7Wa0>jT3PZCdcwJWRCWH%GDB`-L9e~_o4!fMCZC6rhTSJEBa7Ut$f+` zExwvqc3Gxo)mLPRdj^-D*uivkH@1d2j`ei`ypO&0K6TCt8UL7NBZ+j^!W))GxvvpS z&go}sy^Iv1rBRN)zX&Z7LgPHd1*M}5+3Ei2Xm6>N>0kPz4F)RC_z~Z|lLSSJDB}o+wY%<+bM4=1N43U@obFHJPT)?KqA6hw zaRK!^CMwQoR~H&nDJB5>|_j-_~IrmXg#*&dv*sQ*x^luaVe?F2dNw3zr(h zaKtwPI{vdSVDH?We|u?}{qfsU*Q$m>q;AzBe&wirbmvbf#nG=oX=FSumVuAAXVm8l z*i%N)cH`W$0plF9{?&c#DBu>)WBlft-4a$LaV!9u7(Gz87hI_3*C?uSvbNP zP>-rxrY_#cFwQYkra3?2hoQ749*(QcaOz?}J@rlq<#4CI5w~<7X&E#-K&8 z$Oof%#D-%zDtmHeM}dC*ulBNVTUArRbR-7e$QMD?=jTYxPs2d&EMR#=n({`E+ z9&ZDLOX*?=vqt>bpyX>|I+Aq^p|(maB%-JHl0|&(|WQd*C1{(m>S>8 z_?8pg=$_Lr+){Rnh6XGJ_jXe?dPT}*6sLJ^bOl$}97GJ5(3_Ldp2#>}9o`Jg9T9a6 z*Ty*O_w$%HI;kDbG%pS zdWqa#P7dja_&V_1vc^|(@_bpNR>_Eadwnq__mzL3=tOGwknsCv7`UN=^VvfSNjG(v zXYeG{sq@e^LOu;jebf>6$A*RMQzgGnN95=B%lK~MxQRa=`tc$r4=9+XA94&mUgcvM zJnm&n_7WIZbJgWVQ%C6wOQwld9wj?xb$P%&NZ+l$@~ORuX#e7cKo^_k@NCjlGFa!; zyo$YxDM_)*!A<@ok#`d%?s8TJp`$Rpg!YPC?F$L+ATN7U&uhbQAU zN8o)lr_AB9eg!X)-DNCq6HmIGV}vjM-XgB>Mpqe1!(-_zo_g$Lrbs5y3Yyl;OG-kV znaLt>RcrdX0?+Ql+wN@Ib;q%(HVrMQ-6hHD`oLKUa9mFKu`32b2k@4kZ|d%fDKtzk znVDI;Q*12JuLpfd|6^P@+4iT!h%=J0D|H}Z04*M0dfJPf^5zK4CW#uJtj`T31y1Kz zVBg&!mgDA`CIf1D2FpxQHOzM?D`PP%I@em5<~)#bVcQGn|M6%heta8lYH{cyYPF8u zootGyrA?#yeg@llF+{#}cJPaK`AVKP{m_Fnz=mXFy@+M3Dx|#fsGpqZ!Ot(puJ+3} z+%P)>CNMrp-q3Bq9h?^RuXo}!rX1=d5ynz~)7TU$moU5s8L~z*ix46^63* zYZm;QcAE;01$BEia&DFUa<{J)z61pv4i%Pl{ya=(j=C|`H3>tnLP~N7$L!tl-bdEUjq&aeVLxXW%7^4$l+O$IQ>h z-2CQ>_~U=iI#Pn|9lNt(J*))u1PuyBcBT|l`E`;3(ozUZv ztULZ&oagWX-S3*VJ?N|YTv6^^x|I?a^Qi<2(qV>{ z&U#E|CY6DBh3Bzx-G0?ZU&VS@O!RI<=WD{0M{}ReVa%If5l^P5o_~MJ@L2lux}yG1 zy*XhWo=h?Ur?fw~h+X$%=2FF=zN)wT5%s87el_+rX(n<=$=iB8p7OyHc69WHuu7 zf4V(W_~6GXV(hu|H90EqS9gFPtD$+mhPI32q?W6$j&n^Pr;ojFaZctNoqKJPy6vo< zRFo+?gGeKHI1Z7_sJ+x&)aS7yW_Q6Fg2LR*{3q&@lTGsYh8-9!gnea#MJTmzq3DBJI@sEG}N3Ba+iJ85OArQWZD(_s>x=;RVQinZN)D@kAnp3TA)E(w zm=^htywWUWn1k5uV6?!mH{rSGZRN}6Hy;U~Cv+2&QfC4%BM0mvHF}f9_BBo+$WBM6 z?+Fo>0y-T@Nrqo70#{6Ot+wS~Ou&&Y(BVMCPHeAc@j- zz31D0XWkdO*Sx0wVdY--o(x^semu`#lrTqO&o8iqFqd{5K8mmR(%gSEgL<`@xJ~V; zX}ZqIo4^j8oXoybtLe))iT(Eeoa)ft$Nwksa0!ui;=_B((q}u8X2`&OvusTbdu?rI zLhh|{nmt3Q)OP}Ltot-N#v@q%neoTTu7uQu4N*LDl{HIu+`mrgJIS92MSB176)}m; zk&x;=C&cMfo$E^anZ#yEUbWLRgo7W)6g5|C6kau17SZ^^H%vD|Own}_t2%RO@~7&& zH=~|>8+5eksr2v6n&VJHH+rym_nE%umyXb$XV$i5BiUy%JW{)7P&(@CdOAL`uJ1ln zi=y82W=R_w%dbU0JG`hWwBjH3nVAXXQmN7r6&##awSFyD8lar^doaV}+TMLg2{&m_ z^la!tkhM8Ls(y}B*rJMg_3QTsL!W{W;pFXHsv5-Hm}|+Z7WnUH;o7D8w8ylE1(Ytg|4{d&s~T zsu?Tq__cRjiHF&|-oY_&Qa#t`T@3Q8bN;pIbf_IR{<^d;>`%TGR1u9GJkv$YabltZ z-7xc+a=_E%;o#yd5;jzNwaOrAp0nCEGp|k&TLD5jg-Dq|13UTg2jJn!VpY%ngPpai zyJgEKyx3<>qX79rC*@S;g5VgJfVLyeY(#9~Xm6DQM`!ZePV>7^E;4co<+r8B%&492 zG5zly%wuyKQFcSfA?bG-1x%QE_;j)!&(SxB-PV@5wV>Z@G(KfyKnO;^HyfGPo`=%u zOPMa{)TIJ{CbmyC5Cp*!mXV`xqr4*s!bHC(j$Unh?Y4`xke9r`8< zrPm{(VMnotkyc%sOz1#&9_qWvju+~t7a(A%tLmCXH4}TSSEfnB_oh*EQa!fptbM7_ zvVB4EO{`VItxav_*`IB{GS@fHE|-_G=$3odju!4;y}mii#^kwaN6*&Cb1mx8;9=a! zUpB|&>0#)*FPwW)WW}^7ckS?A;YSKV;Rugj@-;Ih9sgT$Ph@Kf1#lmDKqnbw=UfMM z{uhC&2a)knrAsiAsX^bq#GKkEWoTNK)6$Esa^yGm=Zvd*HCbHA0SvW_sB#8=*}o#k zb{&=*%$7%;Rk_yq|tyz-vg z)VLib$QC_Qobs`Njby>A&!bn|@rqp2x!X8&Iw=}g$2>NRA0Oqjdv9W@@?!!BXUA*IC?M4bP7VWyg zSW{*L@{+rg5lsBz6Yth)U6?BmHwcpX@hJ)}MTA#`Y(>)9C(^cLXsNB-4$7ozj%GQy zhHFkdhe}$GSVcvoXJVhLyS|cA0kEVMhKPk%_6M^+tBZa3{_gHMHpWo z%ImQrX>(#x)j^KWZS3;Ia(s$TynV)$9+O&Is!SXYmTJ#m!;90d-A_R@;KuC6zXxeV zLPT|_6OQkDdtVsau@Zk^InpMag~eRdah~BQksYB%g zOYFGZ#4RSwl`Kfk{R9wCLI<9Ut6qQDWgx8ccUls<7WdS$CY?=inF`LWeI>R!`{gcd zZu?~T`k>!|#C--ZVL5$B6%Y12St-Ry>Dj=}Omo@V+2TbdOwj9|d10-50si0KH$_LV zAgqSw!K9~swT~P+ZGQeLl9iroDeTa2HBHM+HP+LOVX1r|}h4 zb(GBq`X{fBRA?uD6ii>h+h2Xpy@Hl0&8Y2WjyCiXuvLABevM0ESLL_!R!V%kEVp0e zI#uDD?K^j!TH)xYLx@YM|a;wuK~hdWRB zPvEh)%aIomd~GF__S=QAn*%;3pFc`Xkc=#yI?vD`4`usfbxtR%StpdFN(nn)45If+ zehsurat@73#-MD&6m3f3bJbfgQ!A+tH>_>W9--jmbg)Jza{L#*#cr#j;(|pk?ik72 zYlj+gJ}c`jtsK^-C6=4&-ls5W`Nn?5Az}08+QfF$rqBCGuSN)}saMKsX{(C!Ft<#! zi65GBxLy%?bgxi>2FamINB6J*P@tL-=&YJyLCulTbeP|^UAC${LhOgq)h@GRfl0lh zpo_bM;@w~&UuSN;`k(c38_f~rqT=NbEJYvgij_#CFNRaO3sg*{ej%fGpO8{MK+#hrVb_N$MqfXCYis?(VT~z3iSDAzJ?Hp;vzl#7q z2!aZjndqk{!0DAx!)+%A=r)`T0f#Wh!2~FN1z)>!K;@NT+2~r;*H+lw>aA{Vc z^N4srjc=xJhWgERf^t9kk{>3o%*7fs{505mldBM{EEs~U#MxXTOMph8P^HV)9+fy~BV5fQ&}scwIfZdUZpCS}wVG7mx1_(hh*AuO(BsmnDNWV`>>q~x zCNouW`(}L)6gSGYMdht-2RaHn1O*(^7>EhK zmvm%ZGYY`4i%(J|NhTsZVj)j~@NvR_1uN<>@_5778K>fpBoy%wf3hUaxzaiT;sz2V zX5xEa0O(OrPH=qW-(>#5Sk`i*tI>A!>YXI;p&{zAjuLkw3t_LZ0a~=@`cDj@%WzUd z+oCcV(JVU*&pSf30D}W=#M17s| zd`$G^81Qy43rl2gN3oAi*-vlyX>==~&UbOiJ_816je`tj2yZ_7`TLyFAqazwU(`>d z*zXd8DbyiD>BH;KkV&52gZ;l_H$7HA$AEN&CmHbb!Ft5Hv^ z{;z5OLFQs|CD9m&de-s6kVN1&CWXc@*{v%H95|W&r-l@Fw8>Kxbk=zq;M%_r1M;{G zx> zAD!Qx*#JMDF|x^-!8e!kdzMHy93KAtSz&T!fNg)7Uy%m;dP2(t27V_B_Z7F<5LNAf z=l)sKQ9!*(4>}oo!+3uBejM7MyHMuw z|7R5lf#iw*E2|(2E9;FOY%^hJ$NPE$@Zetm51{1y$5v!YgA{WLZ~ zmKEDbF9r`M76Ba*7J-=ClL+L>_;{J&9MSi@WZBm3WzbyTdHaFo?X zMx)P$i+c!56#)18>|2Fif2%uB3g7|@?j1@qIUK91YV+-Z#~<-BKo5I;0t4mXXX=BF z;3dRQ-aGttdU*5UuEH&4Dsb(nU5Q`Rqs3&Z#moa8sow%8ResdmJd_*{jDQ~Bh5lyN z4ISzAaF&Xh#?l5Gk!Iin$KS@^k89YtlF+!KknN%t3~pNg2%h+7_b*U1C){bk$vY33 zCC?@Dtp80MlBttQqhN3llBhuG8|oLiNy7HnF(TbSaSK27Tm(5pU6GfZict>1fa!&xmWr^BkOPcyK|6!|99vvS7NY z4<=i>2MV27XJqDIOklWgd&I2o6u6w*gC*Ym$O|Ojk+8=TJ&UGvh@!zUAMY8E~}2z>ng84&82p1)`g~@vDH*4R5M zqJ3~<0dTwtj;$OEKX(mpDbqoR2H;PO#g|ttG>-a7dSKPy%395$hjzk{#COtw$1wkP zkHEI)TL@ccXb{)K2c$+X7=_lqhO|bf{OSj1$Rc}shqHKOMJ6=oW9w;HrKmC8qll>F` diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py index 26fecf8b59..f8f5967545 100644 --- a/unsloth/kernels/fast_lora.py +++ b/unsloth/kernels/fast_lora.py @@ -75,12 +75,12 @@ class LoRA_MLP(torch.autograd.Function): i = h @ W ### Backpropagation chain rule + See our blog post for more details + df = sigmoid(e) * (1 - f) + f dC/dW = h.T @ dY dC/dU = X.T @ (D @ W.T * f) dC/dG = X.T @ (D @ W.T * df * g) - dC/dX = (D @ W.T * f) @ U.T - + (D @ W.T * df * g) @ G.T ### Down projection LoRA weights dC/dAw = dC/dW @ B.T @@ -95,6 +95,8 @@ class LoRA_MLP(torch.autograd.Function): ### Gate projection LoRA weights dC/dAg = X.T @ (D @ W.T * df * g) @ B.T dC/dBg = A.T @ X.T @ (D @ W.T * df * g) + + Don't forget to see our blog post for more details! """ @staticmethod @torch.cuda.amp.custom_fwd @@ -141,13 +143,7 @@ class LoRA_MLP(torch.autograd.Function): # DW_dfg = (D @ W.T * df * g) DW = matmul_lora(dY, downW.t(), downW_quant, downB, downA, downS) DW, e, g = swiglu_DWf_DW_dfg_kernel(DW, e, g) - h, DW_f, DW_dfg = DW, e, g # Inplace replacements - # se = torch.nn.functional.sigmoid(e) - # f = e * se - # h = f * g - # df = se * (1 - f) + f - # DW_f = DW * f - # DW_dfg = DW * df * g + h, DW_f, DW_dfg = DW, e, g # Down projection LoRA weights d_downA = h.t() @ (dY @ downB.t()) @@ -167,8 +163,8 @@ class LoRA_MLP(torch.autograd.Function): d_gateA *= gateS d_gateB *= gateS - # dC/dX = (D @ W.T * f) @ (U.T + B.T @ A.T) - # + (D @ W.T * df * g) @ (G.T + B.T @ A.T) + # Final derivatives to backpropagate backwards. + # See our blogpost for more details. # (D @ W.T * f) @ U.T upW = fast_dequantize(upW.t(), upW_quant) # (D @ W.T * f) @ (U.T + B.T @ A.T) @@ -176,9 +172,8 @@ class LoRA_MLP(torch.autograd.Function): del upW dX += DW_f @ upB.to(dtype).t() @ (upS * upA.to(dtype).t()) - # (D @ W.T * f) @ (U.T + B.T @ A.T) + (D @ W.T * df * g) @ G.T + # And add the derivative for the gate projection gateW = fast_dequantize(gateW.t(), gateW_quant) - # (D @ W.T * f) @ (U.T + B.T @ A.T) + (D @ W.T * df * g) @ (G.T + B.T @ A.T) dX += DW_dfg @ gateW.t() del gateW dX += DW_dfg @ gateB.to(dtype).t() @ (gateS * gateA.to(dtype).t()) @@ -217,12 +212,12 @@ class LoRA_QKV(torch.autograd.Function): V = X @ Wv = X @ Wv + X @ Av @ Bv ### Backpropagation chain rule + See our blogpost for more details. + dC/dWq = X.T @ D(Wq) dC/dWk = X.T @ D(Wk) dC/dWv = X.T @ D(Wv) - dC/dX = D(Wq) @ Wq.T - + D(Wk) @ Wk.T - + D(Wv) @ Wv.T + We then sum them all find dC/dX ### Q projection LoRA weights dC/dAq = X.T @ D(Wq) @ B.T @@ -275,8 +270,7 @@ class LoRA_QKV(torch.autograd.Function): dtype = X.dtype ### Weight projection LoRA weights - # dC/dAq = X.T @ D(Wq) @ B.T - # dC/dBq = A.T @ X.T @ D(Wq) + # See our blogpost for more details. # Q Projection d_QA = X.t() @ (dQ @ QB.t()) @@ -296,24 +290,21 @@ class LoRA_QKV(torch.autograd.Function): d_VA *= VS d_VB *= VS - # d/dX - # dC/dX = D(Wq) @ Wq.T + # Combine derivatives to find dX + # dQ QW = fast_dequantize(QW.t(), QW_quant) - # D(Wq) @ (Wq.T + B.T @ A.T) dX = torch.matmul(dQ, QW.t(), out = X) del QW dX += (dQ @ QB.to(dtype).t() @ (QS * QA.to(dtype).t())) - # D(Wq) @ Wq.T + D(Wk) @ Wk.T + # dK KW = fast_dequantize(KW.t(), KW_quant) - # D(Wq) @ Wq.T + D(Wk) @ (Wk.T + B.T @ A.T) dX += dK @ KW.t() del KW dX += dK @ KB.to(dtype).t() @ (KS * KA.to(dtype).t()) - # D(Wq) @ Wq.T + D(Wk) @ Wk.T + D(Wv) @ Wv.T + # dV VW = fast_dequantize(VW.t(), VW_quant) - # D(Wq) @ Wq.T + D(Wk) @ Wk.T + D(Wv) @ (Wv.T + B.T @ A.T) dX += dV @ VW.t() del VW dX += dV @ VB.to(dtype).t() @ (VS * VA.to(dtype).t()) @@ -356,9 +347,6 @@ class LoRA_W(torch.autograd.Function): dC/dWq = X.T @ D(Wq) dC/dWk = X.T @ D(Wk) dC/dWv = X.T @ D(Wv) - dC/dX = D(Wq) @ Wq.T - + D(Wk) @ Wk.T - + D(Wv) @ Wv.T ### Q projection LoRA weights dC/dAq = X.T @ D(Wq) @ B.T @@ -392,21 +380,18 @@ class LoRA_W(torch.autograd.Function): A, B = A.t(), B.t() batch, seq_len, hd = X.shape - dY = dY.reshape(-1, dY.shape[-1]) # .view doesn't work on non contiguous - X = X .reshape(-1, X .shape[-1]) # .view doesn't work on non contiguous + dY = dY.reshape(-1, dY.shape[-1]) # Must be reshape + X = X .reshape(-1, X .shape[-1]) # Must be reshape dtype = X.dtype ### Weight projection LoRA weights - # dC/dAq = X.T @ D(Wq) @ B.T - # dC/dBq = A.T @ X.T @ D(Wq) - # Weight projection d_A = X.t() @ (dY @ B.t()) d_B = (A.t() @ X.t()) @ dY d_A *= S d_B *= S - # dC/dX = D(Wq) @ Wq.T + # Get derivative for dX W = fast_dequantize(W.t(), W_quant) dX = dY @ W.t() del W diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py index e8d6b36f15..2cf3acb928 100644 --- a/unsloth/kernels/rms_layernorm.py +++ b/unsloth/kernels/rms_layernorm.py @@ -27,6 +27,11 @@ def _rms_layernorm_forward( n_cols, eps, BLOCK_SIZE : tl.constexpr ): + """ + Fast RMS Layernorm kernel + Inspiration from a Triton tutorial: + https://triton-lang.org/main/getting-started/tutorials/05-layer-norm.html + """ row_idx = tl.program_id(0) col_offsets = tl.arange(0, BLOCK_SIZE) mask = col_offsets < n_cols @@ -49,7 +54,6 @@ pass @triton.jit def _rms_layernorm_backward( - #dX, dX_row_stride, dY, dY_row_stride, X, X_row_stride, W, W_row_stride, @@ -58,11 +62,15 @@ def _rms_layernorm_backward( n_cols, eps, BLOCK_SIZE : tl.constexpr, ): + """ + Fast RMS Layernorm kernel for the backward pass + Inspiration from a Triton tutorial: + https://triton-lang.org/main/getting-started/tutorials/05-layer-norm.html + """ row_idx = tl.program_id(0) col_offsets = tl.arange(0, BLOCK_SIZE) mask = col_offsets < n_cols - #dX += row_idx * dX_row_stride + col_offsets dY += row_idx * dY_row_stride X += row_idx * X_row_stride r += row_idx * r_row_stride @@ -71,15 +79,13 @@ def _rms_layernorm_backward( X_row = tl.load(X + col_offsets, mask = mask, other = 0).to(tl.float32) W_row = tl.load(W + col_offsets, mask = mask, other = 0).to(tl.float32) - # row_var = tl.sum(X_row * X_row, axis = 0) / n_cols - # inv_var = 1 / tl.sqrt(row_var + eps) + # Get saved row variance inv_var = tl.load(r).to(tl.float32) normed = X_row * inv_var dY_W = dY_row * W_row rowsum_dY_normed = tl.sum(dY_W * normed, axis = 0) output = inv_var/n_cols * (n_cols*dY_W - normed*rowsum_dY_normed) - #tl.store(dX, output, mask = mask) tl.store(dY + col_offsets, output, mask = mask) pass @@ -92,9 +98,10 @@ class Fast_RMS_Layernorm(torch.autograd.Function): X = X.view(-1, dim) n_rows, n_cols = X.shape BLOCK_SIZE, num_warps = calculate_settings(n_cols) - Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = "cuda") + Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = "cuda") r = torch.empty(n_rows, dtype = torch.float32, device = "cuda") + _rms_layernorm_forward[(n_rows,)]( Y, Y.stride(0), X, X.stride(0), @@ -120,10 +127,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function): n_rows, n_cols = dY.shape dW = X - # dX = torch.empty_like(dY) - # dX = dY _rms_layernorm_backward[(n_rows,)]( - #dX, dX.stride(0), dY, dY.stride(0), X, X .stride(0), W, W .stride(0), @@ -133,9 +137,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function): BLOCK_SIZE = ctx.BLOCK_SIZE, num_warps = ctx.num_warps, ) - #dX = dX.view(*shape) dX = dY.view(*shape) - # X, W, eps return dX, None, None pass pass diff --git a/unsloth/kernels/rope_embedding.py b/unsloth/kernels/rope_embedding.py index 99a7a50f47..2bf7c1b272 100644 --- a/unsloth/kernels/rope_embedding.py +++ b/unsloth/kernels/rope_embedding.py @@ -28,42 +28,35 @@ def _rope_embedding( BACKWARD_PASS: tl.constexpr, BLOCK_SIZE : tl.constexpr, ): + """ + Calculates the RoPE Embedding quickly + RoPE is Q * cos + rotate_half(Q) * sin + See our blog post for more info + """ row_position = tl.program_id(0) head_position = tl.program_id(1) col_offsets = tl.arange(0, BLOCK_SIZE) half_head_dim = head_dim // 2 mask = col_offsets < half_head_dim - # TODO: Fixup int32 locations to int64 - rot_position = row_position % seqlen - - Q += row_position* Q_row_stride + head_position*head_dim - cos += rot_position*cos_row_stride - sin += rot_position*sin_row_stride - - Q1 = tl.load(Q + half_head_dim*0 + col_offsets, mask = mask, other = 0) - sin1 = tl.load(sin + half_head_dim*0 + col_offsets, mask = mask, other = 0) - cos1 = tl.load(cos + half_head_dim*0 + col_offsets, mask = mask, other = 0) - - Q2 = tl.load(Q + half_head_dim*1 + col_offsets, mask = mask, other = 0) - # RoPE repeats sin and cos so 128 = [64, 64]. + Q1 = tl.load(Q + row_position*Q_row_stride + head_position*head_dim + \ + half_head_dim*0 + col_offsets, mask = mask, other = 0) + Q2 = tl.load(Q + row_position*Q_row_stride + head_position*head_dim + \ + half_head_dim*1 + col_offsets, mask = mask, other = 0) + sin1 = tl.load(sin + (row_position % seqlen)*sin_row_stride + \ + half_head_dim*0 + col_offsets, mask = mask, other = 0) + cos1 = tl.load(cos + (row_position % seqlen)*cos_row_stride + \ + half_head_dim*0 + col_offsets, mask = mask, other = 0) if BACKWARD_PASS: - """ - Q * cos + rotate_half(Q) * sin - is equivalent to - Q * cos + Q @ R * sin - where R is a rotation matrix [ 0, I] - [-I, 0] - dC/dY = dY * cos + dY @ R.T * sin - where R.T is again the same [ 0, -I] - but the minus is transposed. [ I, 0] - """ + # See our blog post for more info. sin1 = -sin1 - - # RoPE repeats sin and cos so 128 = [64, 64]. - tl.store(Q + half_head_dim*0 + col_offsets, Q1*cos1 - Q2*sin1, mask = mask) - tl.store(Q + half_head_dim*1 + col_offsets, Q2*cos1 + Q1*sin1, mask = mask) + pass + + tl.store(Q + row_position*Q_row_stride + head_position*head_dim + \ + half_head_dim*0 + col_offsets, Q1*cos1 - Q2*sin1, mask = mask) + tl.store(Q + row_position*Q_row_stride + head_position*head_dim + \ + half_head_dim*1 + col_offsets, Q2*cos1 + Q1*sin1, mask = mask) pass @@ -90,7 +83,7 @@ class Fast_RoPE_Embedding(torch.autograd.Function): ) ctx.BLOCK_SIZE = BLOCK_SIZE ctx.num_warps = num_warps - ctx.cos = cos # Don't need save_for_backward since a view + ctx.cos = cos ctx.sin = sin return Q.view(batch, seq_len, n_heads, head_dim) pass @@ -99,8 +92,7 @@ class Fast_RoPE_Embedding(torch.autograd.Function): def backward(ctx, dY): batch, seq_len, n_heads, head_dim = dY.shape dY = dY.reshape(batch*seq_len, n_heads*head_dim) - # Cannot be .view since the problem lies with dK since - # K.T's strides are incorrect. + # Must be reshape not view n_rows, n_cols = dY.shape cos = ctx.cos @@ -122,10 +114,8 @@ pass def fast_rope_embedding(Q, K, cos, sin): - # We need (batch, [seqlen, n_heads], head_dim) Q = Fast_RoPE_Embedding.apply(Q.transpose(1, 2), cos, sin).transpose(1, 2) K = Fast_RoPE_Embedding.apply(K.transpose(1, 2), cos, sin).transpose(1, 2) - # We need (batch, [n_heads, seqlen], head_dim) return Q, K pass @@ -155,7 +145,6 @@ class Slow_RoPE_Embedding(torch.autograd.Function): cos, sin = ctx.saved_tensors # Q * cos + rotate_half.T(Q) * sin half = dY.shape[-1]//2 - # We reverse the minus sign for R.T RH_dY = torch.cat((dY[..., half:], -dY[..., :half]), dim = -1) dY *= cos RH_dY *= sin diff --git a/unsloth/kernels/swiglu.py b/unsloth/kernels/swiglu.py index 63418a82b8..037dcda84f 100644 --- a/unsloth/kernels/swiglu.py +++ b/unsloth/kernels/swiglu.py @@ -28,12 +28,11 @@ def _fg_kernel(e, g, h, n_elements, BLOCK_SIZE : tl.constexpr,): g_row = tl.load(g + offsets, mask = mask, other = 0).to(tl.float32) # f = e * sigmoid(e) - # https://github.com/openai/triton/issues/241 exp MUST be done in f32 - # or else Triton crashes f_row = e_row / (1 + tl.exp(-e_row)) # h = f * g h_row = f_row * g_row + # Store h tl.store(h + offsets, h_row, mask = mask) pass @@ -59,23 +58,20 @@ def _DWf_DW_dfg_kernel(DW, e, g, n_elements, BLOCK_SIZE : tl.constexpr,): g_row = tl.load(g + offsets, mask = mask, other = 0).to(tl.float32) # f = e * sigmoid(e) - # https://github.com/openai/triton/issues/241 exp MUST be done in f32 - # or else Triton crashes se_row = 1 / (1 + tl.exp(-e_row)) + # f = e * se f_row = e_row * se_row # h = f * g h_row = f_row * g_row - # df = se * (1 - f) + f # DW_f = DW * f DWf_row = DW_row * f_row - # DW_dfg = DW * df * g - # DW_dfg = DW * (se * (1 - f) + f) * g # DW_dfg = DW * (se*(g - h) + h) DW_dfg_row = DW_row * (se_row*(g_row - h_row) + h_row) - tl.store(DW + offsets, h_row, mask = mask) # h - tl.store(e + offsets, DWf_row, mask = mask) # DW * f - tl.store(g + offsets, DW_dfg_row, mask = mask) # DW * df * g + # Store derivatives in buffers + tl.store(DW + offsets, h_row, mask = mask) + tl.store(e + offsets, DWf_row, mask = mask) + tl.store(g + offsets, DW_dfg_row, mask = mask) pass @@ -84,5 +80,5 @@ def swiglu_DWf_DW_dfg_kernel(DW, e, g): n_elements = e.numel() grid = lambda meta: (triton.cdiv(n_elements, meta['BLOCK_SIZE']),) _DWf_DW_dfg_kernel[grid](DW, e, g, n_elements, BLOCK_SIZE = 1024,) - return DW, e, g # h, DW * f, DW * df * g + return DW, e, g pass diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 34906a5a1a..8a7722fabd 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -13,12 +13,11 @@ # limitations under the License. import triton -MAX_FUSED_SIZE = 65536 # 2**16 Solves https://github.com/unslothai/unsloth/issues/7 +MAX_FUSED_SIZE = 65536 next_power_of_2 = triton.next_power_of_2 def calculate_settings(n): BLOCK_SIZE = next_power_of_2(n) - # CUDA only supports 65536 - 2^16 threads per block if BLOCK_SIZE > MAX_FUSED_SIZE: raise RuntimeError(f"Cannot launch Triton kernel since n = {n} exceeds "\ f"the maximum CUDA blocksize = {MAX_FUSED_SIZE}.") diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 95926a14b8..d9e5686be5 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -20,6 +20,7 @@ import gc warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch") import bitsandbytes as bnb from transformers.models.llama.modeling_llama import logger +from transformers import AutoTokenizer from platform import system as platform_system platform_system = platform_system() @@ -115,24 +116,56 @@ def patch_tokenizer(model, tokenizer): pass -def check_tokenizer(model, tokenizer): +def check_tokenizer( + model, + tokenizer, + model_name = "unsloth/llama-2-7b-bnb-4bit", + model_max_length = 4096, + padding_side = "right", + token = None, + _reload = True, +): # Checks tokenizer for out of bounds ids. # Mainly a fix for https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha # where had token id=32002. # See https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha/discussions/25 - special_tokens_map = tokenizer.special_tokens_map - max_embedding_size = model.model.embed_tokens.weight.shape[0] + # Seems like the Fast tokenizer in Rust breaks things! - for token_name, token_content in special_tokens_map.items(): - if type(token_content) is not str: continue - token_ids = tokenizer([token_content], add_special_tokens = False, return_attention_mask = False) - token_ids = token_ids.input_ids[0][0] - if token_ids < 0 or token_ids >= max_embedding_size: - raise RuntimeError( - f"Unsloth: Extra special token `{token_content}` with id={token_ids} exceeds "\ - f"the maximum vocabulary size of {max_embedding_size}. You must fix the tokenizer "\ - "or else out of bounds memory accesses will occur." + max_embedding_size = model.model.embed_tokens.weight.shape[0] + added_tokens_fast = tokenizer.added_tokens_decoder + added_tokens_fast = {index : str(value) for index, value in added_tokens_fast.items()} + sorted_keys = sorted(added_tokens_fast) + added_tokens_fast = {key : added_tokens_fast[key] for key in sorted_keys} + + for j, index in enumerate(added_tokens_fast.keys()): + if index >= max_embedding_size: + bad_indices = list(added_tokens_fast.keys ())[j:] + bad_tokens = list(added_tokens_fast.values())[j:] + if not _reload: + raise RuntimeError( + f"Unsloth tried to load `{model_name}`, but cannot succeed.\n"\ + f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"\ + f"Fix your tokenizer since it'll perform out of bounds memory accesses." + ) + # Try slow tokenizer which can fix things! + tokenizer = AutoTokenizer.from_pretrained( + model_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + use_fast = False, ) + return check_tokenizer( + model = model, + tokenizer = tokenizer, + model_name = model_name, + model_max_length = model_max_length, + padding_side = padding_side, + token = token, + _reload = False, + ) + break pass pass + return tokenizer pass diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3e6aa36099..88c095265c 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -135,8 +135,6 @@ def LlamaAttention_fast_forward_inference( Vn = torch.cat([V1, Vn], dim = 2) # Grouped query attention - # K = repeat_kv(K, n_groups) - # V = repeat_kv(V, n_groups) if n_groups != 1: _, _, cached_len, _ = Kn.shape Knn = Kn[:, :, None, :, :].expand(bsz, n_kv_heads, n_groups, cached_len, head_dim) @@ -210,7 +208,6 @@ def LlamaAttention_fast_forward( pass if past_key_value is not None: - # reuse k, v, self_attention K = torch.cat([past_key_value[0], K], dim = 2) V = torch.cat([past_key_value[1], V], dim = 2) past_key_value = (K, V) if use_cache else None @@ -219,7 +216,6 @@ def LlamaAttention_fast_forward( if (not HAS_FLASH_ATTENTION): # Xformers memory efficient attention # Also has Flash Attention v2 dispatching - # (batch_size, n_heads, seq_len, head_dim) -> (batch_size, seq_len, n_heads, head_dim) Q = Q.transpose(1, 2) K = K.transpose(1, 2) V = V.transpose(1, 2) @@ -231,25 +227,18 @@ def LlamaAttention_fast_forward( K = K.expand(bsz, q_len, n_kv_heads, n_groups, head_dim) V = V.expand(bsz, q_len, n_kv_heads, n_groups, head_dim) if hidden_states.requires_grad: - # Xformers does not support backward, so we have to convert - # GQA to MQA by cloning K and V - K = K.reshape(bsz, q_len, n_heads, head_dim) # A copy will be made - V = V.reshape(bsz, q_len, n_heads, head_dim) # A copy will be made + K = K.reshape(bsz, q_len, n_heads, head_dim) + V = V.reshape(bsz, q_len, n_heads, head_dim) else: - # Xformers does support the forward pass though Q = Q.view(bsz, q_len, n_kv_heads, n_groups, head_dim) pass A = xformers_attention(Q, K, V, attn_bias = causal_mask) A = A.view(bsz, q_len, n_heads, head_dim) elif HAS_FLASH_ATTENTION: - # Flash Attention - # (batch_size, n_heads, seq_len, head_dim) -> (batch_size, seq_len, n_heads, head_dim) Q = Q.transpose(1, 2) K = K.transpose(1, 2) V = V.transpose(1, 2) - - # Flash Attention v2 auto supports grouped query attention A = flash_attn_func(Q, K, V, causal = True) else: # Grouped query attention @@ -714,7 +703,14 @@ class FastLlamaModel: internal_model.max_seq_length = max_position_embeddings # We check the tokenizer first for errors - check_tokenizer(model, tokenizer) + tokenizer = check_tokenizer( + model = model, + tokenizer = tokenizer, + model_name = model_name, + model_max_length = max_seq_length, + padding_side = "right", + token = token, + ) return model, tokenizer pass diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py index bb6a68f59e..3e3d5956cc 100644 --- a/unsloth/models/mistral.py +++ b/unsloth/models/mistral.py @@ -92,28 +92,23 @@ def MistralAttention_fast_forward( # Attention module if (not HAS_FLASH_ATTENTION): # Xformers memory efficient attention - # Also has Flash Attention v2 dispatching - # (batch_size, n_heads, seq_len, head_dim) -> (batch_size, seq_len, n_heads, head_dim) Q = Q.transpose(1, 2) K = K.transpose(1, 2) V = V.transpose(1, 2) M = bsz * q_len - has_sliding_window = isinstance(causal_mask, xformers.attn_bias.BlockDiagonalCausalMask) + has_swa = isinstance(causal_mask, xformers.attn_bias.BlockDiagonalCausalMask) # Group query attention - # if n_groups != 1: K = K .view(bsz, q_len, n_kv_heads, 1, head_dim) V = V .view(bsz, q_len, n_kv_heads, 1, head_dim) K = K.expand(bsz, q_len, n_kv_heads, n_groups, head_dim) V = V.expand(bsz, q_len, n_kv_heads, n_groups, head_dim) if hidden_states.requires_grad: - # Xformers does not support backward, so we have to convert - # GQA to MQA by cloning K and V - K = K.reshape(bsz, q_len, n_heads, head_dim) # A copy will be made - V = V.reshape(bsz, q_len, n_heads, head_dim) # A copy will be made + K = K.reshape(bsz, q_len, n_heads, head_dim) + V = V.reshape(bsz, q_len, n_heads, head_dim) - if has_sliding_window: + if has_swa: Q = Q.view(1, M, n_heads, head_dim) K = K.view(1, M, n_heads, head_dim) V = V.view(1, M, n_heads, head_dim) @@ -122,7 +117,7 @@ def MistralAttention_fast_forward( # Xformers does support the forward pass though Q = Q.view(bsz, q_len, n_kv_heads, n_groups, head_dim) - if has_sliding_window: + if has_swa: Q = Q.view(1, M, n_kv_heads, n_groups, head_dim) K = K.view(1, M, n_kv_heads, n_groups, head_dim) V = V.view(1, M, n_kv_heads, n_groups, head_dim) @@ -133,16 +128,12 @@ def MistralAttention_fast_forward( A = A.view(bsz, q_len, n_heads, head_dim) elif HAS_FLASH_ATTENTION: - # Flash Attention - # (batch_size, n_heads, seq_len, head_dim) -> (batch_size, seq_len, n_heads, head_dim) Q = Q.transpose(1, 2) K = K.transpose(1, 2) V = V.transpose(1, 2) - - # Flash Attention v2 auto supports grouped query attention - sliding_window = getattr(self.config, "sliding_window") - sliding_window = q_len if sliding_window is None else sliding_window - window = (-1, -1) if (q_len <= sliding_window) else (sliding_window, sliding_window) + sw = getattr(self.config, "sliding_window") + sw = q_len if sw is None else sw + window = (-1, -1) if (q_len <= sw) else (sw, sw) A = flash_attn_func(Q, K, V, causal = True, window_size = window) else: # Grouped query attention @@ -317,7 +308,7 @@ class FastMistralModel(FastLlamaModel): tokenizer = AutoTokenizer.from_pretrained( model_name, model_max_length = max_seq_length, - padding_side = "right", # MUST be right or else attention fails! + padding_side = "right", token = token, ) @@ -339,6 +330,16 @@ class FastMistralModel(FastLlamaModel): internal_model = internal_model.model pass internal_model.max_seq_length = max_position_embeddings + + # We check the tokenizer first for errors + tokenizer = check_tokenizer( + model = model, + tokenizer = tokenizer, + model_name = model_name, + model_max_length = max_seq_length, + padding_side = "right", + token = token, + ) return model, tokenizer pass pass