From 055885bd5ac35a25084574beafe6f63e23f5bb7d Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Wed, 11 Jun 2025 16:45:28 -0400 Subject: [PATCH] Add pdf support to file_search for Responses API This adds basic PDF support (using our existing `parse_pdf` function) to the file_search tool and corresponding Vector Files API. When a PDF file is uploaded and attached to a vector store, we parse the pdf and then chunk its content as normal. This is not the best solution long-term, but it does match what we've been doing so far for PDF files in the memory tool. Signed-off-by: Ben Browning --- .../providers/inline/vector_io/faiss/faiss.py | 25 ++++++----- .../fixtures/pdfs/llama_stack_and_models.pdf | Bin 0 -> 37844 bytes .../fixtures/test_cases/responses.yaml | 10 ++++- .../openai_api/test_responses.py | 39 ++++++++---------- 4 files changed, 41 insertions(+), 33 deletions(-) create mode 100644 tests/verifications/openai_api/fixtures/pdfs/llama_stack_and_models.pdf diff --git a/llama_stack/providers/inline/vector_io/faiss/faiss.py b/llama_stack/providers/inline/vector_io/faiss/faiss.py index d0f6dd6e3..b1326c06f 100644 --- a/llama_stack/providers/inline/vector_io/faiss/faiss.py +++ b/llama_stack/providers/inline/vector_io/faiss/faiss.py @@ -9,6 +9,7 @@ import base64 import io import json import logging +import mimetypes import time from typing import Any @@ -19,7 +20,6 @@ from numpy.typing import NDArray from llama_stack.apis.files import Files from llama_stack.apis.inference import InterleavedContent from llama_stack.apis.inference.inference import Inference -from llama_stack.apis.tools.rag_tool import RAGDocument from llama_stack.apis.vector_dbs import VectorDB from llama_stack.apis.vector_io import ( Chunk, @@ -40,8 +40,8 @@ from llama_stack.providers.utils.memory.openai_vector_store_mixin import OpenAIV from llama_stack.providers.utils.memory.vector_store import ( EmbeddingIndex, VectorDBWithIndex, - content_from_doc, make_overlapped_chunks, + parse_pdf, ) from .config import FaissVectorIOConfig @@ -292,20 +292,23 @@ class FaissVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolPr chunk_overlap_tokens = 400 try: + file_response = await self.files_api.openai_retrieve_file(file_id) + mime_type, _ = mimetypes.guess_type(file_response.filename) content_response = await self.files_api.openai_retrieve_file_content(file_id) - content = content_response.body - doc = RAGDocument( - document_id=file_id, - content=content, - metadata=attributes, - ) - content = await content_from_doc(doc) + + # TODO: We can't use content_from_doc directly from vector_store + # but should figure out how to centralize this logic near there + if mime_type == "application/pdf": + content = parse_pdf(content_response.body) + else: + content = content_response.body.decode("utf-8") + chunks = make_overlapped_chunks( - doc.document_id, + file_id, content, max_chunk_size_tokens, chunk_overlap_tokens, - doc.metadata, + attributes, ) if not chunks: diff --git a/tests/verifications/openai_api/fixtures/pdfs/llama_stack_and_models.pdf b/tests/verifications/openai_api/fixtures/pdfs/llama_stack_and_models.pdf new file mode 100644 index 0000000000000000000000000000000000000000..25579f42582739d5a2462fa8075e8b1842bb9181 GIT binary patch literal 37844 zcmb4q1yo&2(k|}72@VH$cMt9o+}+(>f)iYVYk=Tx!QI^@kl^m_@J^CD|Ky)}cjmpd zSZkl%yLa_h)%8^ur|OU?h=|cK(X+yl%^dEp!!ZFE0d|I#aJ;+>$`;PnCICtqYXcht zfReL;krlwe))?^nUe?ao#M+4pjzPiE&e+As#1TNLWMyH%0F(x>VPfF|h}+qj0bduf zGjakbn%EkfIGQ+8@$WK3+$oXr93EL@!Y`~W9sM-u}ZIQNWWo#$x0CWlu&`DzAzg2M>H zPQgw|x0*M-AJ7gV5eC?iNd-4J-w_D(EycMuw^)W4D|t1G6!v_#G)acK^@a1NKWA!;L>*Ybu?%OToG@6K{ zyx-$xdPnR1z;~EW-#WO(TD|j)cwf($``tSH3r{;7J}te(f{F77G`3g zFT+_)zV|&ZbhM%iJHGPTp#nM#{Ms7~-?UWLsuxw$x9#FkfvHlkqg;nhx9$gpt8HrS z-4ouBJnaGRjtPIrqBtwEsSREat1Eza`IiGeK@>9|FNF&ibc<}io7^N(Yq0c6_=Iv= zt+HVKB}m-}VrAlCP(^2x3=xHNzv)NR?Z;wHfK*A!t@RvYGHJo)Ix~zLJ}vbG@8yRH zHJuqKGD=D6kv$Lz&}-H(JQ)88CwW1sF{X2PqLP-9jf+6-2~rVbp#$urwgHzHE#DMk zFfb?yt{Qpe&9VT3`~1t~r@N|`mrISlV+|Gl7Kwz`bp17Lc;g3eWtVhEa#dz)IV)HI z2-B*C@w5Bku|=h%RODHQ zCTfM45K$rtSt=6owM~hXbCgV}RCKNpuGCn!L9=(7RI@H6pie7tAca2SG5+zUZe=Ai zJ#KF&pq;C$`UgSM^Yh}@Cv6p)zOX|WO-dLAx)_*nL>F7kh#!Wh(w8KDaD|_j)wxpWg@*X5r&7yRZTv@+%y@Rr zpa=RT%b#2G`mRmE52D8IeWfZtmgtCYxKKo>#3j)hXVMyrD|I&wm1daqHp_JwB-Uy# zwR-D_k?-EM=v^J(@4&~PEP1&#H1%REW+AjlUA*U>&y_(o$dri>^OTvI1*|L=bMmMX zS~*(6@c*cc5?Zh7-EmUmZB&BXb&^Nune~-aQSw(@cX?EqQx5w=Dvi+UMlRq+L2433 zoI>}Jh2uV7v6|Q{0#9d{>|4o9NbyRoF8;xty!$B27kN@iBEdKpHEXG19ky_nzUJZi zXEqPjoxM>hSmo+6BmCz`Wkr+xoiIMBK>k{zS%I}>hl-NZ@03>M8dS1P8_F@psas?S z*l@LY(0GMu;(n_NS7K=_*Li!T=`1c7E>|arDvp-)+2FYCN#*kI>72gj)mW?Kd+O!~ zc;lVZ+z1JoZlgu^>yC0erqykj$9C~GY_=nx`ro=~_`M@-Lhb+ZnxPmNV7Zw`o4_|L zDG2v9cFgykp7`(%(&I@)bo<{H z7DR@PJ|u^-Jch3NX5rk*$R-AeI|Vj3b-bPt`aQY%^db2@Jo>$!>j}Kv`@OteJ%?uV zdv{`~pm;vs6t~s9`g~c+O8!tIkEXLgLa>Mwk#sD%(CB2nfNbje1mV;EqGj;SjJc6p zEPHg3foU!l*_y+taB&eSRZ<>AM|r|`mt=~oa?2N0MHFsJ0)7D!1uPLK54#d;!5Lp? zTy4zQ_L~UzC%J@-gD?OvUP zM>V)m_#bCunlZWA3IbC^ENd7wZTp98Izvh<0yPz>dx`bAtu|Rkd64;wnY4; z*@cs9rB(Zs^3(j$-3%5KY&)7yp@Q4=gU!rkQ00Oh-_23UbFw1|>#pgj(@l_XPt?kn z1g);jWeFYh$MIbnG5O@P4^(0|rGJsP#B?N$sMcQmggfhaERej(+s|$B^*DK`dM{JN zg2>FliA-7Yt(k!X>z^MqjY~$G8s%UNVdNg{(v$Qd9Qx4 zO-1RC@5~&k$G%JmCCAQ8g!%H1Y|Nl!wWdg)Vu{l3?ik+E@yP^Oh6P^^yPWqO8+NNb zu`C6nhh4w>=wQUMuk*Q9vvCbi*noXbT8-lXCQ@_!sosBzo`!sUNw-X-*bLx4fWkGK zzf;ktR&lOL5jBPP&g5B$o+z=sDE9}qMIOlQVTpvxHJ*`UtF2DU^0u=Rz(1*;$JRoz zC>PB4%ekhI2>M}~7aIpJOWjrB7qKR40CtVVrw%ys))CufAvN-*y}{0}>@G2&jYJZp zTHUS&I*++U;yuXVXy-z(#BV@QgU2QVTV4}fZH7OCRYXQh27;r@6%;J4a~}j&Haed* zJmYy@>xT)xk4l%5-<}B}sImW&;!5HV<|>GQw|v0OhC?c$ozTd3K>n1Oj6PI~Z1 zE?XqS4{(e~xjfM+{M&5=gKM_UUc@}pstYsY+JyH?ZY7ys5gdxMS5D0^-o5C|X*a&(nDcD!z9GO}S5{2^9b`6>aQ07!22BB2z zr6s&Hhm*2(&`XWD$3uoCM4=j>~uWYJQrnniSm+&tqx7R4pGEc4CuB1{2oyl^38Xm0li@G zeWKC<@u*!0-X8Z$I$2M$1Pz506M`RMHKS=WiVYot$Qo!hn$`LZ-6g_5lX5A|>~WHY zmT-i)Acwr6)4Oc0?{viZq?LXEaa+n4sYQouNnV@ut&S-FfH(p_2e({n&0kKuD~XRx zJd%#!tD|CU5Y1*LZ8{uyA^X=QwfNQoZ42V~`-t537cX+cTKq~(yWUikq~ z7}%$*Et0z;aMr_~@?!zD6niw|O|SjD7u@JpocM+ruc-=d&Xo+xppm(h-AKl0n1h;X zCzCfPZ}4=huLD_*x+Wd62#YSSw;Dp|*b`$(s9vf}vlW$woXuB_J)@Dq<{~+$bzW}w z(u;upo|`Hf)166(;yoK3E6qWBQ+0<1@&nFs-m1yIxSBzdg%Y})0|M3Yo2s7P2@7W% zkp<8V>=<8nW9HM^AZqHAo!Ye+i)?ga)8|L`PRHT7X}A)4gs&wgdEotVntbi`d|#N`XZa~ ze&aofaQeP|QqKt8K3)a}*Vj8fkpwVLWLTmidPwYj;xR%iEd?#2tjJI`9r4Fete)#* zXKL%&OZ}LpQenCof*3-=PeF5m&qP&C=5;wO5}uR>Y7K?n&&DPHC0n*if+({T)f+vv zJclgZV)d?RbNI8!2elGg)U_2j(4s3%4YMsp7ywI}iI^@d%?FJ1X|IBpJ%>Sb!*WBq z5n15@@0PeP?J2Q$Gb1GN`yW2wxdk1H9o%bRgDRx+#(ki|Y=hgNzcOL2B1#_TGPA5F z47DzbF@I}y+jFfO5WQRFyF0Zm7sU=j$6SBGi|cNWZNs?xkSt#T`260mwKhqapZxzD)Kb0PJ)P9295 zucU<{HB=J5F6PeA5HR41BA4>+W273aug~Y4nOjf!BveijoN9TRVD9)(57maCx^YwW zQN4IBqy3Q7onsiaeb;WD=Igji9O*1EjnPuM^9KTO4|7T%!AR&5s3IXaHR5$@R%{c* zC6iU_B(`Mb`42Xey~9F}NDSDTOF@xzP;#v9IN-a9J2Z)%WJ_R!LxN|Qa;?)4wl{Yf zB{Ib-gMylpp)#K>#7rmZRj%7~Fg)w2KvCX8{ zwes59oxj;mm@t`{dKoqW8U7YaO?Y>N%BruDc%#ftaNldewe?Apcig0|Vn_6W3Z1Y1qE*!d-<%pkjii zQjMsx!Vq#5K<@(VYZ-t5yF%(?{pQ+qm(Pyn0JezFseMMvo6wfh2nZ}A!1y_$Xq0ft zPP!B^P7t%;(+HnN#us{oA9d{^88uY1A)rLQ@#$7H@Q5 z>7C#}Odk&GcA23RdZu_~Y@Z?3#2N`W3k`+kO~x>|k{3VJd62CK;lwtirKDCY!{N(y z=#;x9waUyv2W!V5r#GFY%)-y)C(@g@t<3gf3jw}&lXK|L%MLV+ab?nuhNB{zn~3ll zyixC+U)P0}ND2ygX{<|>xO6?pkVSSC=h}V4^6K6*sw0|0F=Ip%(*4DIzof-hweOWk z>7{_-Aoo*WdP5QXa4cfaJMW%paI8vs8N3PF5=)A(&hE5EYU&_nH4?$Q42$`NGdJcA zbCoi%yy*OevF}&rBX82u#MQXdG*>+Z*V8fUI2bH4(`|I38r67qRVYSKWuF$dxH=YI zm+ZdpX^X@A*lmi!FVx>=x^$Y_29K&Xj0}@)5tE+x6>wj49Z|vt>RLmE@7Up<+Ei@ z>Do*Nbu-5v+B^7Nz)>?q8fSk%kp$^~Zd3hi(-AetbB?hWQd#q{)+}F)^dB@?7r< zAPnwORR#pP-p&j+uh9k8Dc6seA0LBE-zH{wf#mBYp3o7O&!UuFmC`(N*z6j7s->ai zz!^;aB0NMS{x~Yf)BE`vr!Q0|)l0iL0dZp<494WkL3$P2)!>f*0rgs%+!a9km{V?6 z(NJ~xWi~uTf~k$#7R#79CMPl27=w<+stO@7_$f(+JEdj$Cc-?^4nKEq6|!_L+!XEc zduNmG$Y&4;*{{?NI-lhM7R@RR+??0WDldM!Mj|H7Nncwkpvb6az95*w4(nAEsmIx{ zp1{Y=rw&_^vK{Lbm$Rr;mt=YR6lh2Eie`BXOUrXjwfmY~)UYUTjG!f~6XxPj{zz<5 zJuB>2_)63rRItN1pa(xcXvLdTWrmzP#CzMM#gd4?b}(UnW*O4*n7mCa74 zLi@q%c4Uh&p~d^FcDk8-5)T+=Y~haHeTREgz#Ex&n+KbhXrPKon+m=tUaK&N9podZ zWMZ$Aaw6$cE3+~fx{G!qsj)~ZEi>N^fJ+>ib`@7vjudI(p5ciRpQxKTf$NtkAm37* z&n?^Hq{7O*7>kFXN=C@rdDsz?k6{ZGADdM5XK{}9N9|>Zdc9ezYDZbiUFTi9#4kL) zliopI69p6T>Ek|fC>HdKgaYwyjk>V+P*_#)jU{gl=kX^~kW_xl-G$4q4fk;y!mpfM z_u*9mG`_lg=1c-3zwgNAst6)^>S3VV3g}?oW}ga8q8@TpIriG?bWM|7bZQk8>I+Fj zO6x|@=7M<-r&1r%c->WCpt~R(riuITl9-hWn%^^~-3YuVO89+)c0K(ZV{z zY~iF6a#z*G_pmdxbvGTqX{A#*z9F*#Hsp}KH(}5x+E# z;N+(Bia{ICEUj;2vUUP0u7(NI$xTFuBlL4>9J?aPs z%(ZcWBi?HA*1rUsk+r_g+){5w1?!gJO&d%ixO^|X8%aaM0b8y{{gr)xiuSU|nbQ2H))shqMdXXr?{SDUQfm>~ywItkUg_j1ForAf7oWzQ{DP#Ek> zw0Rs{&+{4_O3ZPVPU6xY?8dY5a`{hqA)jKchE3I`cp?wuS4+t@-nwAzntz=P#k@RE zG@PJx34P#-D9WIVtX_VYpOSfZ!EM@|UnqgyC&JP0q`J+Lx|`MVJy3S3bNt=-OLsV% zy@9S9cAX;%@zA*aiDt#JKru9EuTNI@bAIsEsdvHCh(YfP&Uf+zgc0e4Xt@f75mKI+ z;C)6z#NwRPJA`N>^anf!uV6~WON2?-Uf^U@!x5?a6KV#hB8rvSIXqil!J>(oD!u$4 zE<)Di#WL~bpPS}VsvEo;QHL0vFWRFyn?KFRlx(D0Z#;)#2+3NzzSjv&&nWWVY?=s- zK+6!(B#;T4)db13U~V8}8i3Mc_lorSj0*9Ht^7$?EerC!6PwC#J;T5K~k;TK4jU5mTNZ zy@ikQU*QQvron&KSNni-$RZ<6zLDbz@99R!>Hu?FgI1AusEk_YpwP6sm zvvvMCCdLY2{XO<`RMy1U!a&H*9iYVs9Asx=25@jO1H}jfXPejpzmNG@=23)WP;z#3 zF>+QgaQrpM2*;poVCV#3Py*JNevJXx|61oS#eUxYqg3@jiaGt1>jH!UEGbD_7&`&9 ze(JBN^UtCZ$3I^)r}uDlakXUOa0t<3~fM@z?!vCf8Z>xz~o7ey?2w;!^>a76a`l&PUgBjqL z_CQkr82@Pjray~(zbyB!LYV(X2-6=z{>`ucMTo-R$YB0M#vfI{-@f_(D1+q>8GjTB z|F1G+B!2>7`sI-S+a#=iNclHs`WGaBFNN(7DS!CZ|0fpqKcsN}mp%S_EF6DG`RN1y z$y5Gb3b2^^k238qv;9XYf9$d}K#S=YmjO(_s0v{EX%9sJ(@%wfmtS7_hch$(S7-h^ zKjiu+LgxSGhrjh?{^|4oLJ6inIKlkiT=74M`GXbAKi%v<+Uu7${$J7jL5yF||D%}y zZT4SAPz3x^0(kl5>wg;hzj5U6xXSbg(U|`mMgBJQF#SO^=Kn^KzlHJ#&zS!kL;hCC zA2ehBZv^>UA%C#!zsUf$Ke*Tc&$FM~^@{~RL$TJcC=MjVFP{Ve55!;e_;rl_6{rBe zxB=YC-&1~`Eh#@Th zcHI0v&y^ibfO&|Viw*FQW(52~{pUITH+%f^Ek!#!phczs=a~DI9{hYD2TWX~O+0=c z{{LJ9@C(4t1mz#SCx4Rk-QaElk`1!0r|i6DMF+^xFWx^8yZfCJrV}4mJQg zCnr5KBO^NyC-5-;D@9@XwTZx!SoE*m1uFP6nPT~6?LQ#=$;f|)AY@`|Vrl_Qjs7zP zR!$arMrJmS-+9WvL-_4PKyZ#Gt^k%_QQ;2=e@482hwwKmv$C^tva$j=SeWQJ85ueL zxAMR2^jrC#J_roAzm@-W8UWKn3rCUrTKSInuumV{AQdbq|&L)mPH!uaZVcGtb^#2z9 zZ;yaZZ{cKbZQucnUB6qv{;`;hfvp*U(!}=9Hm*Nd`nwy83Bbn0#>x59On=|8#mvUW z4h+TrY}q<$gLPLHo#*f^Icf9pI$h5EF|$vWc@pa^B!nsf=C2JfV5(stmmq~1B(H&R zNY1GVquit1WDnq`rjL|;hcJi`$;?e3mKPmOQ`8s@lZTM^uDL^L?mJN{_vxx#*6!tN zWXxQ#UZ$q)cTKrW?!uE}L}ZXbA>_}_<~q;7UX(KkkT!HBOay#B%N4fJ(~QVeXwX3) z;Q?8!{MW!%sC~|zs@r9<+Z})ve`XB0 z&o+2zu#kQWeR|z??*@XeTZ!%06_9$V+g!FU2VzFj&ydDd?*iH=7<#>+?_shNoUCK- z{i7$L>R=zFHn6r}qNQ#^4C2hu`%Tsz`B$~2DaX~?v^0(M$I_9P@-R+k?LYSNXrjHk9T zrCN*~E)F%w^CUh*BYIWl{Q-lxB(C6hF9`Z~;03TbkZ`JiA<}8i$Sw9e9LqHhfKf9T z86%4LDHAP?fi}oV{C(Ltp}l(>FMtUZipGB994(oKcRLd7>p42#Rc{Qd%hc>85$AD@ zD5jA8PA;|+5vNuEeFVULeYOu%@XR?A;BhjGfPFrWEeEH7^XtHYA#fiL@L@nrSo@0{obOuym!I+ z#vNChh6q&2(6O$GZxm3)iK+Se#88`}z7M3d2Sq8Dl!Fn9g6|+#RXITNa3bs65v>JV z_u@Sv4+aYL5-o7tTRU^N-aUDu1$Nol`7}n?MgCwl?^s^ykUV|bq_i}u#HT&K zdl_jtuU+|9C*9Zb1Ors7@-Qo~op)Mkw~E?bx>5D zt_%T7iyQFOF`IGXX8NrdN;}}289z7?+w5fu4e6y*ks{Wa)jxa*SX}i;;YS4_2im9R$+(#iwyx`k)mqu7kdqiL_#0hLRG|z=L=x~T5o<+ZCPuxga z)vR&ZSbPSg>Q$s>#H?nksrv=KHX3aOpl{ldaa7tiko&49mLS3x>_O}*Awh3_7$vrb z)aeXP6}|%d%0t}_yI_Hc58|2e%FO7E5mUy8brA$yri`Ek88P|KsM5Eh3Bp^rZA)ROo3U%{=@VXMhhGk$2;z1(8bJpT61Bi?H-_6WZw!{4)+`rcGe5v2{9&Tw?=@nm8nrrlYkc5sK5h9cgF<0Ds! zHo3A(9YE&OEv2+lc}QP;Jp(yHe#tlzk=zNkxLZnol6ug(&{+{A@3FCm#shra%j(2G z!duyOh3OE)_AQ*Gjo%vGg0&Weq3>c3YZN~s@iIrS$_C9wE_x+!-qHd4tq^gXQT`?hnGlg|CQe>Temh4qRW^o2!LADUfF(~fLm??CcO9=Nh94Vr5SgCqpcfG=! zgct6Wm|an{Kh|l8){#s@(w5vU)@KXj0vlgqV5aFRj8R&h@<|sjDBHoX)V;h;s9Eq* zBbTX~<2<>yBi>{PgAwZ90K4IZy%E~00sN*%>y8VA5cMNg7`A{QDlyt1#$8j|io^*? z`e2VS?vHJZ3!?iBZ`l_~IYsm{W#L+ju$>JG8O$l^z49yDW&tLm|7* zV&5G?O`-+RVnIZ#A;SfK z7OG_wR;Se_FT>pEmhSfI4()bWNB51-)kO$vGE|QIDBMT2(YL|3QMge!ZM-kKZxp4= zB3~;s^Z7n>MVtT-lQb|mcA@Ff^3>b_*>s8R4wl6$d?-Lki*_(Y-|{gjg5ZJK2m9Uw zP6ks zt4Q_)^ZKm)46DIc;5m~~h?hEwtkxb<8#f>bxm)-#cX_>4-- zXrH*v2U=6GA|6Hq?Zq9*wLi5&hQB8$lhOUiW*ZP=Kxa)nf~J=9<6t21lAlWpN9a~s zIKI!B%_kx{?{FeR5APVpp$GHs9$yS9C#p`<8?9f^#!>d7a_~E!LiZ2Q-5w2l7_WPb z9|kE^(8FjX`$=2K2B0^$Ad!;%pAeAwMx}JaZU*Vy-zDjn;Pr4tC#Y3-sC6Q6vp1=y z#HB=Set5u_`i8-ow9s>%X1fEE{`fjJCB7!L(~{^1u?fz!k}8R&5oWzgu z+<4`}^YWc9|KpIH1#ww&Rv`T3YT%X6hIo~vQdFC~P799;;=Ga1AzRE^B-40Bh((Gk z(u5b{iKr*DuG+A968UPFU^R*v-C)k}C)9ct;QEVbmemZ@1tPHg0=8wkccnUgYP;KBi0GTN;x2aX-Nk2Kgz4Vls- zq1xnVtVrMIK!hLk2x|tMn@ej)hJ?PMF+D$1mPz?W_nC~KH=oEX=_9n3d-Yl>nF_5( z*DGalfmDeO*+_3n1OF2PLO1!2$C@ zQQ=-aF4qq7jnVZ0dXa-dxtJJ&8Fjs6d^k1?%)X&Di}lM*WCHZBz3wPjqjqQu2+<=? z>|IM`9QMNY(o7*P`hUDhrO(tTv>vw$C2N*HpG(dV*B&8QgB>K+_gZWP-K zv&z%)d)t?)B2a3V!TFkegkAg;;;U8*&0w9NPtsZQNdLc<{2R|~}fx5O$t zW$nL9tR#0rJxUza--y*E@W$+5AHU{pQ+#3UT5RJqGi|W!gTkJ;?ra?IaK%Q~{>b!L z2Mu>Jh~g075PQg*K6_Mj}=;$6udfxi~3#;>TyOx^wL~inF^qL@-|2bV(A%^1y=W zPck@ttW1do7pyxek=VTL)jYRnWJ}5j<}#6(p4L&#;FU+HaBBchYV^7RM=+isZ=%9B zy?4jR-C8!$%p;wKL}9nI>sTTgF)t2REs!^O=HY7t&X5jN5A0Vld8<(`7*{wTB1yO+ z0))>{SI`>Ku_Slj0zpmm+t<3R_>Lg-?7L8|B(K0AkfkSMgaja;HLq|dt=zeimDJe}2jqx^LXA~%N8p=?Y;bu|KbEuT5iBdQ1 z<|wD=eS))zt6_h8AZy|U#SKZ}I{opD-Uo?2-74gOf^)GhP=_n!$$TZ{t_C&+%?5+7 z05FuIclz8A@m>(qwlzs_W@kwGz}R6XEBDrw5&2r@tm&2+&uD?ahKly(@bvc$%SPIR zuKGk^Y!s~?;S2uoeeX6;AV60bX=4|}_ebGrU~oG$*`C#uykH0Dv+b4mz68asAEkK@EOJSYrOiccik7H8VkD8uNmBHm?;R zT-4z#QFzy%V&TDavJ9J{9@Hh1uGQ!-9U z>J?1*!;5X{+}NxOuYeR(F>C=Y*-@c{=v5&_E? z`FNR-D(FquoA)JR3#Zn0NXui4Fhs4uuW1g;#jhz>tl|cUQ4GO_ziQOo?UF z>3OMdx%qTL-!3GyVwFqqS!elMGoY_|7rA5UxDg(a-t8NGCc>LSzVL=I>9*~C-b>&u zF#N1rY%?|KI zdLY?L*ke=Y#svP5CJ*wL{iybmbAdk8lkNn;7ks}@&OeT;8gF|=`4%WD6#pQ^Tw?6K zx0DPvdD%5eM0Vr_#d95MhMY2zwQ6~NU`oU5$klO&Ob%{Q9>-LDY#>@SJ${AFD?P=e zpsigDKL*wwpaZHSOk2E!NQxLsRgfm;7AiX_7-KCj#!!9!Bt0xy<6EAne`fD!#W#Yx zaW~4drpiF#`_$9o&h#$CxH{;UaH<$vq}G!b{xfgM(6>siqC7BAJb|ryhCuBT7?6Mvk)uJs+q8O-NJFIyt_5U~u}+3K1WE{d zPq3PE3SbHF)2|3#6_rb@3zFJeeF!-Z+)$|wLFD3M*TuF98_AonSrHnPl}*7ZEVWN2 zSX~s1r?v24uZ4F7A)4RjFs-c`xWG+M1NO%#9#+)9sZvEMegrj_G&-tt$`;)1UEqkc ziWG>SL1!=$VJ<8tdZ97pJObz?r2CiKHtRfy+VzEm$s~ax$*m1fv_XeWCtg+nGD;%n>CsIvEX?=Ia->%6406T>^$p zI3XH_nFBaihD#xc5fS3n$l5o$YbLt(4b{x&psisOp`uUxxpPP?4HJwbU11tOKuf#{lF4m}@y|b;62$FWu<^}2 z+^^;a3TOv+yem>~Jhv8-jud79g*EyhWDk@2ND);nQbRPf% zdVV)j&)XqQDyDD4Ii(qu!D0+SGTA5);}EC1Po2mK3?_(65*bF6ZHCUrViA1RGT{R; zV=4&#UVLT_G@oCH^*h!8lqm>u5f*Xy6cmgqfF_FkXBQ83(G{2FBZFp4f%R}oB3 z(U@VtbG3}Jhbr9Wd;V{Qn1M(`v~5{5N|{6roj zXcz61gK0?ii9}UU^a8hNXA#qr2NT@9MU~3^qNUzs_63GF17Jw-B z1U)P0*XxP(ffK?qLEM0{D6B^xuwjoCQt1JMu(WBev1RyZe42vM~E z`cS1l?SO;*+YNVf34#z$pApspDkH)jR0jB!5DV9MfNAfNz3v4|3OJp_#I`$; zjA#b*uF-=i;~NM50Z2x^dn(!>?63%ZazmPdK1P~*SZ#z_l;lplUTI{35NQMY4HHJB zbH_%QbGT@=Ztvbweb9}$Dn<{OBuDE&)n1b-taF@3&Y`R*dcoG{O~*#kd!n)r(kN=( zCSj#(j=hKL;akvEI29tTpcUPp*un@qqwLu>WUJ5}=xBwtL*0>WL#&YP{H;*@AQ#Ax zI?M6w0=L5s4ahbs4KQ|oM7x8Wd^kio4VoLeU8ipZJ-1a4yu*#Y?UW6})u-DKQxA1# z*zmF^T#0lSSP8h(y2IE2x3{{`0g9Dw2S@7i?3F#U>J8=M2m4CG59bEphjJsb-MQmw z#65Sodt-=qA_#noy{-qUx4a6)0e2x7eDlIm|sZw z(89XMV#pmBum2sfwYWEImdFY;ei%a!V(&Aao_}VTpTAC6W%p@9WtSE3dUyui+W|aS zIrQZ(7|zHCI=W`U?7CRO?0R&2-8kFf{e--cvP8WJv!vU>pg9SGb`u`g!QDa6h22rk zNuop7$a-T82sbFsm^TW~q%KVEA~$!>Fnbx+=C=6ncv?g5$Od}OxHr_!#4gA}NmYroQao8b4472o(?>#P3Ip z$LovS5UTw@AEANoK{v?=0H1Wd3Z=fxzsfT|U+Wn^Ffc!N z$Q8VPJ=}k7V}4%U-v=%W97Yk)OKx86P4#h~l$sX%~vm+7}qWIBLsV-+~FYeCok$rk%Zb+YX&sSg97`qm$z{$8= zt}t;a865kduG1)F=6a2F2I?Lg>=D-gf9@N9*(? zR>iL#lDoz`qTEKhWjfdPTomiGRUQU)Vl=tE2`(LF$*+U^L^LB$;FpGsRkHX@nMTa1 zW8`^|iL41-U~kd%M}weT)%c*$opAzn(Oq%_@&)sfuejNTf8iSkK@0_poia^boEpSp z!C5-h1{65Sm`3zjD=s22VU_2kbQ!Xzbxo z8`C%@wO!%Ur-i386I~jBWivBNjbWzhJQJ^gzHWAxUgzjB)33{Ah>_oD>Fw3KY?W#{ z-I@vD=_!{Cr_4uj3L!Jc%j#2V>}z}<_@D7Iz_@&O_dTZa=<-aULBhLzzd$o(CM0g# z8!u@$de@+1Hr90dQ>_W;#2CIuFV7vU(YT@9yLwx0=(yH!HFLYayJn4y(45+w;n+A8 z7<~-Oj%&&!v&)G`pWJiORn(L8h5)(|Pou)NV2ugrLgK@HXP&*OEub zA0KiKLE>#@JnMEGd>BpOnKC!ejm^BYxi}dJ#6cW+KU7q0ezn2qshJd`AvXmBgMa{oa@Z8F9~1fxX#d6MiG9zfCEt#| zY$Ilq^l+yq-wwLv_=L>Uu^Zpltd7Q4JIsJ~p9m0UO(oU0(;7#mWr76|Bn3_vm+xUU5*kVW&D3!a)fI>QIzO4KYdt*@7az;O^*Hc#u@U z=Slh1!tB69&^K>XCS5>=8o;47ql&&T{oG_F)%iksjfr+FI+`OSpOGAS4F^K=y^_Kz zMoslSO4>Tjy!mfJ&RIoyjMF?wW;d9(gpGz+9JV0N`0qBvkmw2fJ!@8dO;g5;%gTz2 zX$ixohO@32t57YeX)F~5YWj30vV!@%n!Rr7O!};ay>c%xnmylM(Tu)s-xj`*s6i!6 zl!sR6vB5*cBI=j5;tg{ZI6d4FvKINxF-G1n*nKc~Hc)sy5VW^d z9zqWk5*E~q+=7yds!F1g#c&B5H^uNW{_Wk&|?rB7A;HPJW& z$7&EqP+uV1hnn(;>x+~@>+b`I#nV^q2{YH-F~N+x_2=c(=1 zFCUEAC|uchM~PxdN#>~t?n$i=!K&i8`koPTr$=!y46SO){`7b;-e_z9 zkt)>c6Hbvy5uPeH5?X0_vzmp+sbi=P$?omGQj^c>p3`v2Bh2FsT`LiHH_Nql>*>2` z!QDv@&KS`WGf*ACB#JS_4`zf}8oEv}0*9`slSPSbh1v_)Pm!D}rRZ-BDZE1Qz%4-c z-8W#_2Y66a4G$s1ifo5*v=S^gHUYN1Pf^5n+vhu`c!fR1V#h0A?Fd_LKx$?rEs$=W zZ0hCgSaG0bL83ityFRdrx6K<1rRC+Mm0>vbeQ~C16QF zDordXnOL;NbVSN#%#|NvY=FR%_<~3)W+~>p0WsTL^07Xx8jO`h&k++&iw&oVLr#c` zugPZ3YSZu8V94-198?X;m*Xm!YEepCIX5zIt?&dNKq0W)FZON(E6zplq!V5244oY& z#nGCEq``bqey@nT)X>q)#DYA9(I|ax_+zDVW-eg+tF&*Kpi4EFufWLl<7mnzN6n81 zo;{_KxC5OE{dd)hI0%5->~lqudH3_I4VT(?@i5haqC=Q5U!++!2YUCr%1% zi#2-?9hqOMo0qE(l%e+^FtsGTxBT{OtJM#Dk8GbzEwgQ$Pv zrD1yOwoL5KJ+y*-)~u9gyiY`YST4^i>-{FUB40%SVs)V^6y?btpMk;?uXi~mzM-XJ zkXK>n()ND#b4i~Sc3)Ne#Z+KDUBM4sFM#^ee0G1hOBhM~k4 zqlTj{wJQal8qpanu%Td;Hrq!m>*}!QzN*pyDbsSR*+}3EEWxUh)uaCwfzSKAot>pMuLYLeC5aE06!>MgIcP`2a^CVK@?g$AiW z*~RP{Bo1w7*U%@-9b^rk*DrC|a|v*aO;C;UiNXbda;)ZfjkTIhZFTj^o-q+;)?%@s zNZ?=EHznV1w^if1d0jP>R|~nt)qoak zH|H^q!H!I7E{~UREbd54(gK!kU{v&UaRXz^TsK2sSKQ0v1sQ=`%GeWk%Cu291%X9)7bRoY+>fyJU{A``k2|5%& z+ZpTWViu;Gu)WKY5u}XZnHIN@QCx&fDS^pk56czu97a~$!E1CFS(*doY;4~dun)K8 z+QUo&AFi%uLZ0s`Px6LnG7ZtB8gghG+b%7c<&eHVOGDz6n#&T|ZP}F$zQH(nPfzzR zb2GdbDM>dVvgP7ZZTAJeEC>cmSWp~{vJJ^o5Hy0{4MuFn#-60xNM~fcKO^J) zOvdZkEcsh^>xgbAoe8mYp;)F+j1{u8La|nh8btj7^;(N!p_UGEv+5~xa|!9@^3csC zo4YlR`dbH`sXFp_W**PXh)V2|0`^B+lhHPzhi>Al0nGYh9 zGq}2{(hm2Gg|M*c2Sr87k!ZAcMSAy;orn<51x(4ZJIl+``-@G{))M?mS2Ct%vF7duN${))8=cIJ|tup+&*XH+XpsY@aux2y{mR5 zfsj3G|47Eqy6)O3(^g)0Dmh}^s0Z7wd1MpYcXpsCS|LWc4N;{AMF}*rj%sQ|p@wKg z)FDft7AVuiJ&h zK-xpw~&FXf&C%8d)|Xt<@N1 zG~rl}EnF84%0@e8Q9)p74wfbaMWY?B()lGx>(>aH4!S_qf!6rBN(rSqi2*0qpxTT< zm>~;eM~nN#1ER1_Bw`04s$sNok9@!=tTPfLiz^m+zf70PZ8DXAWqISx)NFdtYGgshMd&DY2RE&mH`eDBfDyY799 z{Ns)xxxR8~=ZPU?SE7ziBI|cta^nqbS6Yud=;z2^R>*-uvU~@KI8y}%gJyy#4hc^T zUl4B9U8mE`_FgP4(6t(_m98~t@|`-tonPp8=IAo}k}k8)Fhb4d`j9Tm<}WNPC;(rM zA4fWX3Oi`s9mxd0!kv*pNE}y9MkZUL8Y2_`8dPu82$TOB8 zT%{Qw$GSq(;S5w?fRN-nzmdRE#Wo?imWj{67bTSocB#)2aaxOhVr&VV3xNFp)lEqAg zn6(-if}r-CwzbNwp)zNeL^IPc36-U^P5UNCd{+H>0exF+!{ z(tlaiSq($3xhL@snRi}v@T9)uZ(EbtEbVOCG2^@k%ky`&&D_$`OBic)UNB<#{DPAk zWnyexs1>7j;V1au!5`gb;7Q z#i!22r_RNv&J_)635qwSl^aEKgHdjlK0cgy^RCCq;HTduMg4Y{KmDtRKbk!6 zlNI-WNNKMlC;vz;di7&6Vavf6i#OhKU*gEkPbEHIy^HM)_n`cqgzL)!ImDydf&nsE zn_3T6#cu(vE5pIA3f?Mkg5vSf7f*3nfV2@*#k!Hs6E~m?BbJ!{Jdn8nrHV^ZW%kLrIJv!;? zrp2uzFTZ);^28QWee(mo>PP(g!jYR3FG)KcIcJ}r*uVFoMB?G8Wt;o-s{j1KPySWN zek=Drlx^()P8ndE>d;7jt(JZHP-KIz*ZB>gl^M5{D7FfDyl_@fA2drg#<`KsZeNQsmiymK_`OW)0%7cluV59QAels^$NQuSurvmLZ0pPZ-7 zA~qo}4?;Eb%aFoBYByXl$Tjad2A&qtgCrJLBNs7Z{c?E zA>LE2rKAnwxB2{WEv1~*O&7K0ll&Y$WI24ua`=$tM1y*w=b}c>kJ^ZGnKL1T<65l1u-)kefbsO}N+mY50!Pd~xs4H+eNyGa&sdtWv&^kk{rQ>Z*7Ax}SX`P4 zdlaO?a3mGx_*0?Vn>qt6G$|E9X`S@Agbab2<677VaS%&Dh0$;Tj)G(hV&Xawq}2Hd zTM6#;3i&d#LcYqZ5XN3y?c^)uKJksFZgO+*xGcyWbcO0SaDFiG#bw~LYY6_|YG7ZVmaUtZA|5-v zMRlTy#mp?W9%^vN5tzp{$h2{GM2VrBpJ>TirAAwWy0;s3?`+h)nG*cBs;1Q00gD{n zYP7OJJc@RewL+tqtrc3utW@Y2rB{%r(!W31bb4ah>6cZPi-TF@sTP^f&8|kKdVjkU zNoTc&jkUp6tkevjr|i5Bpub=l=>!CFPNn*G`WOk<=bk;vE3fQf2imb%YDyt}l%xh^zSuP!1&$&|iKCWr zWtdTvZ<{F7*?wX^ZTspu${Ev`Jqb|*jEYLc7}VP;EF2`G0K|++iWapM)^S!FCwL`K z(TQLKg-%pW>HS%gnGjwrumh>sv65q4TSL}Qb*8zStwz?FkMQf+oT5*YSh4Ifg zl*Zx4;TEAljF^hdjl#L&64RyTl_sr$O4@2uAM+^MAk@jKc7$n=S${jdLs&1b*KQIX zk~KDJF`G*zigJQ#jV4p6q(xF|9BUa%R5Z4=TJ|M&_61Od%~Ol5%|>lI=_W9dUhPs) z+d+D%PNPoG4#JTxuYOo!DVBF<>d$fq{i^CQNT1$5Usb5VR~%h|)nPJ{go+ zq&5kqINh|>+7HFDhdosf00wEXq$p=xtch@Y5mudFtgRdVu) zVoXh+ipQGJ%EzUSn+99@xMYaa)u)n#tYIq&6YK3HkDOcT^zxZCdy5B82ASnC4fUz_q}Ra=*~F}*j~M7zOd$QB=V zZ8tw_7F;fOHVx*e)=^obT&lNGYSf*hOt4PLn&g_|p6ER%`v%t?RPp!)!R9yU92rH+ zkx{f9YzDXhaYTctfU?8LuE%|By~&w9Q)Wl3Y%wD71!rkqm~pJk6wX?{HoR>)B*(&~ zlZ8ts3$NS~Wx`3DI+Eg=fuJh>*=- zCbmAlNr4Nr^8%udKjPMMlsTfpzlU>|DRms4I`-j&Aw%H6ez?`Ty&+ta~qy6Kmj-OgZ&39|qBIAc;nQ7H9<*$ODnQ8brS#vONL0(&)keBPG-466Q zhq!+%hiK9q^8Mq2Fw?%iZw00opXZP}eP+NUYMN0{LBmLO34d;!dTcDAD#n;6*4;r` zkyiCu)lzLy`dV#$nUo|8xlA>q4(0V!TT#ujp*O3(Y6bNvB{zV+OOa~ocA(~Ry0XGT zPm0VK=8Vao3xw{atM|Ti`3tX&=rQ5!+VRN6V}&zIf( zMouJe6aU#nYSDw{7jxd zKT+5OcsP1FnleYH#tdP|5WWnN%}(I@b3Khex1$^h$7#Yeu~k?sijn-@LbY$OFkC)6 zr#?`ZHza?Y&?HaJIj6_`Zf9yZ5$86uJ)q8TDTLxlPCrwXTtNQQ`Jhym8@a`_-m>1|PpekNp{Ol8>eWRHNED#JYZJ?QMPUZY0aNj1+1F>&?19I-(&RB(9o^*)7?vvBhWUGLPSM znKi2Gdi9K-XzO~y zVH?j@tEX4`)S-FAA!P8wO8Ss9v(KeZIu*1!4>35#Mgi9ud+@*xn$b=H_R>$rk?s+` z=&~yPw7e{R2A7x5AG5LNY|ixQsKf4ZiY~sWG`M$0CqHSL^5?7OKQeCgU!` z*)J_K-*D5ib4Qhz&+XCgi6s~BZ(aO3TUn*JkM878z}{Cq8ZN(Njv9W7Apg6w1Pwn$ zkbmGT!OAF!`f>Jw|4T<*2W{ONOg-^=QWGS!ggxUVj{?Hqu;`oK=dXjUReT%E|4Oq(~K z_=Y_-L`^OiN5D>usy=FI6dSeAYegsHPG^>*qC)Ja9U`8kU1E7q`phC5fm+$iN*bL# zvvb%pT#3Z-?X36qD9xN_uye4~K?haRr(g%wp4@_I4mwHDIoe5EoC}<7PQm#D3I;zB znW*iM4#bbeb+Xk7t?l3Y8%maZ6o?-2UF8eZzCx?euRR0T6az3aO&;E z7&Mb|Yg%#Zjkaa#A+J?zd3t){lnwdK9Usrzptrq6{@r`mTv zasJc9&ejlsyoBi%^vWGzNhVSzMC!20oKl1av9|*{(-}fn2G>YCQqRc-O%OFH`?T5ap&9u7mT z!)-&|jc|^&(RPkoxm|m^g{FmOxk9|2n4?0fFxE+R#s4Bajs*rwAgrs z!y-ADF=^8x|57c}ENYo!GNGA2i_BL2f`}%YM%HR2J$6^eehbEE&$8JZPM6z_`hCq- z3EV*z8f{hCQ~Wlubhd|mtN z+y0&>yx-|feEH1{+l1et{nxxEIAiF^`1R`N5^Ck}ufP)RqbMLLtN&|~rz7MKbP z(TI0cX7?=|Uf67EE}UbUUD#5(+O(qJPUl_ThfR(inFoLQO!9dc^`PgG9@{-n_1Np# z-{V!sdp)#uPQu3)$UM3g#Ek0$+)_*1Xod!JITp8_bWJkC+8>*J}%A zWBPsOFEVd9A5%l@-B2@sLC36NukV*97yqYsybgMx~R?j9Y-wUMy2J`NI{Z z)qASx#%faSVjY;vs&?@gJ6sWWZVA^QO1Qu&;ex0nZ?|T@h6XgX8mh7LP0^luVUh2j zW^Og`SE)2^ZZ7awtTfyf)R?*Hz@KtxdR289fJ`tiOz9Z0n1Xu4aW1O6q&Bw?V;_Ia zB=w|32yL_(m z_s*YOHgEQ<`=UA5P8jm&@{!9%+RY|!UPOOUasQ@;?u9ors8i4CdFj!U%llW6_j>r0 z9wSPIwVXSu|0QfSt-#e3U}u$(L!MW!Ad=CNC-s)1@kr=>qBPoPUZZ6rx9&6?GZ}%5LA>mk~ld^!~m%!qliG5GQ!pkVQVUfaX8_u zN4VBOY>f2?XFbBY$atF@bWoftNGEaG!&-n}f5SgUdTd-hP<`sm& zuk4DwU8;GCK~4*}=0)j$S!|fb%7ae17?<~qZ#X>XF}GH|Hj@-1X#sl0)>55xVU0 zd$Z?W_{?i>JbewyoFT|d2U725k=^Q*dMcVCrV3M?N$PFy?K_8#*N?T2^Ub8wr5Uls!0Xaqv)=Q3ob|E&i0kj3kGX7d1_CiJ6G#nSCY|J-G|$x2*_ZY}S@NI5*hxWPZRR^Y@l_%lqX6a#9u^WZd zS5&KJ>|OfTi|4$4ZOi(St)0QgE?)A$rYkPJcf~zxPTs$P2&=~oq~;StsO_Z}{`lR3kQ_!#Ef z-jYt(k9);lw!WTTzJaz8-hsX`w#lBczNxl(-l@JzHJ3V$)8lRhoWx>sxkfvg5hyr) zmUYTTg(`}e?bFM!lRm;$Nk+2oQTbFUxS(&#Lb2dVo%f&ldQ<9P+ohUNKjP-0Df2Z3 z6I1Ni$eUPIy8Ob5xQUp&0rq@460Kn2lS~Z?kbslz4U<_#&hnHnO}_)h`9R5&)x5%r z%v`5deM$lZyUn|g&wEN%`1p)F`RwDopSi;{$M~7%AxuLaIKHsELCPNOAL4SMxu$br zP5PDI^t*ig>1{?Vq@FL>W1%6**dyV6^!3#6e^BEK$&aeuJIZlUaNW zvs5wtf*l`BmocSciXhNi2wV4iw)U#3;9pRS`2rQ|fOu54dOfNMS1;K4SxFCmR+7us zGZdvyNxn{>CEL?y$zSkooWH9A?NMZH(i*|Nc^v1Ai8z6zHuBQac$i^{Fx-sQb3 zD%qhH%4sI19qdad?He{^d9Ph^_T=oUvaxmh_X~HfSva?1$T_y(=!dkNzvk2hxbg-i z#t2{F%JV}Z`IFjWFi7?yL&Sczq28|1<#=+63{iVgxZ2RiewJZ~eWKiGm}NMj|EI&; zGhCEEAUq)d?EG~_8;j&Vp*{t*MMDfjLiGjXL*ol(%hN*B3R;TVir&uuEc8|QNWRtO z)HpimmTf(JSu)>>l^~Szjkpc=z$<{3482NKB%jZsug~=v^-f25q`dnVy}r)8FaE8X z&%dA)al2n}5yhptT3l@|u?VMxPAKBi*u|x>t4kWYxHNV-c^>=PA(zIil!nDq(%6-H zgwBZ*nYDgGoy9q4v4un+H^7-Gz?mt)nJJLB+p^zsz>>6x0ZXlA6lxkg-NI$Hg|AVI zm+^`vmw!3L$L?(5vf09Av&9oDS{!2HIX3e2GF_NH{#3e4XfC4p@;%PJS9*wlWsQY3 z=|_PJUF_vUZcF6jLZvAw?dn}_<)@p`?%yW3;BiCQ;Kf(1a+}GL_&Y~0`pb>GE`M;w zI~)J-#T^e`b>*hbmtVT6(K{wmHhofM{036<-tB~}xxMYwoNxDE`lwL&mp!{*dgj?@ z*s5I#AbiG8^={h%PFzwB7kkI#pvn!eh}c`G7j~LNp5$-eJbM4qfWZOxzQPSCY_>_+WA!5`IOrE zL~k%&_Q7ThD79#9MjL9PN*&}EmEe1mNf>%%yY?<9RQSz2g=Qkk&dCG z-Q5+pg|YN&CrBMTlL0bsU#DyUb}Gc>gjr*jBW8^;o0zmHA%LCFEQ1*PzH52PG;lf{ z*07aNy~bf(x$WvbOMcg|?c%wkZ>&K(;OkqOe|vZ56ngK&&=h#ktZ`8quCDjK_nrUmYv?W5O9JqVe`QLrE4Q$$#Yj5*ZOR5^i?U7WQ9e*k zDiTo2l?BQg&9LL~4R(wO|``+&EZvOK(-#k;oSD!gV)N23Zg)3nTn@tCfau<04UJo** zd0}wFi%Xpi_Il2+#k%D>_;|(e-S6kIKm7_o>x0w#14}4;dwaQ{TY8mtPH9Cr_ z%=~;NOVxNN{F{og=SorPOOKvx(U|H&&^n2_&N&8H9T-r%p<%rxdjN1lw#;hT!bzwY zQWAKJ=$u3@9bwqa#3&8cOGK>R(_Q#}W@LHu$nJ)VH;(5&ee12SR&F-NKf-sM+4AzF znN&uSwZwhN(2)Cfd4oQXV3Pxar)I_ZY+)5u_y*O!%$yUaiqjR&)c;#JC2~qoW8&cf za6l*l!TGh0;x*TB%P?}pbcjJFPwsGP8Wn2LlD#9aCo0$z?mF^}$nyd(HY;PvtcvI1 zX?PiT8$QH+BuY<-uu`m+vT|O$Kxx(6^me{ooF=s^cknBO&B{LU&-}aMG4WIBf5orl z5(i^}<9I}&mx@dd6j{!CfJ=_&vmOW&lf}l%kSrvD2JKZ9V15rYT}9woC^jS0qs!9F z7q&c5%?4sh`*=&neJUFwH48kZ_L$UTVU&fCSaq$SjR-Wd=-l&}U}kIuJjg3spEM#w z1Tk~40rT`hT+=_S7(1(&p%_2;#u`miE<8okl*FYTh}RQuDVnE1D?R{bQiP^N7)&0` zv8|u&&?hReTv^8U z=$rqxJ?Xt1cD8$w&9li>-~%`tAgqKE5jOCbpBCo87mpMkyHnWv^>5*}!qUE(sQmT< z?PzfqSpdxpLp{mWi?zrg&Eeywi}b`G-QblH7i)q+(%-A~6_~7NIO*mKJiK&cleZs! zzPHbKcxpFGj1*HS7akQp5coE-IVo`E!UAE9&@b>rt7F7@ik^NFQ}`vs%6=5U9&nPd zQjymG@~qY`Ez^lcRO0~Z~NV59EA57*43x6v8X$acKShJ&Z|L7Fl)y?46 zo)som6I;F#{rhtuBbCROZR3%#%?EJQTNrmLUIypwg)==uz#7@T0g4mYWwpdS$k8!A>B*OS6!gwgcxF?eJ zkjS>}1LVKenZf5cGj@?P!~4>$$92wijwVN~Q^n96&%Nno4$hzxIj_8w0ZRzSoH*;@ zL_}M`GL|qWTGh5_B%z5NpJ8Zy&_b$28Dsbfwj!=r|M+c(XFPU=g}dwKj$8aRU%heb zWeX;axT9||y62Yp7eDewAFFX3(|??=Av3RoIDBzeNsQi4$m<-z`YRlFZ=zJ3op2-^ z*T#zD>kvAWtE^6^GjdgIzA#^zY0Y=$M`p&B z!4gFj^lQ1Rg{$$k+C1(i;U+v!!_gEk*+ftkqg;9e+{>44h`FLwNfzNqGS-m>S|~{@ z((LC_f}|KDd`Pm_5WpeELVyJ@f!|t+SperJXrPiQ zz|KLOU@gpb5;LoFFnVfkc>RsHUU~gB@@3Z-X7d$< zD;!V;f8s9Dtn;i3t%(-jn%?_>W$B2L5v7-uE=X@k%PrBC#iWV?K=))DPU z&XMP8v#t5jxyhdN+u_5p!-==cj)jkv9ZmPABb9v4%9S+nE!O4y1Z%qWvHE#w!BTC5 zi==3xRE(qy6&Ue=A>si;#QkCmLRB5W(84aBf!AO@&DbLC`dFn;T$RC>VgYt3@Dv8B zFubHGro7$6a7;^Jr~x(jNeD0f;H4t60DOBkG)RJ4MM+R=P!jaixk*q4upk1TB|*!_ zHpk$gEa*TIG}n9VoB2CaH04z{0SQyW*}@C=UwP$&{hOXBoO$3oEq!OWC-=Yn`a7?`_8yf6;|jC6qh#JK zkb=wIc`C}Gy4WZ*5v|a~){@rv#Q27?EoDMus4>}EHYPMCIVChDd1L6to!k*?p*wbWTPm_f`%{?A6j2_e3Fujfk zqWAJnCgGOYY}#!$;WnFmyXqVHrsDi#6_YnCzA3TzrWEmfG%x#5T6S(^(+_MGMKyz% zX7A{4PAymnZ9fq?(hp<~$~FzH+jz|jh5x+u?bWX=e7vvX*UJ|_wRP!jPZVY&dDLV$ z3`$!HcRlsc=}Woa?BDPHlQ5_K3 zfT#+H%7DlOL`6WP>8;+$Vtko!rbU^cjH#NMxhb}8;`W$(VG6&C5=V){E?&|IulfrwehK#J5j@#9l)iZ0C zR4%DnQ}rYKbM2AZjYEGt{7L+*_C(E-wL7X`sgBeJ9z-S}Dg&Y_ApH9kgIf@Tn-YWD z6se1J#C6K@)8(41hWSLgx`bDUl_qG4BNHFW62$S=cw7AX_}2KFanX#I$8U{)5a-L| zKa8XJ3xoqo2tTsiqZ_6#w`hTx1rI<3paof-_>OR-k>w<5nI?q8I!fo2qS91}#Cr%W z7Ipk2h%A2MhA12HsbOk)0w${BZYb6`g1)_p<+ftpmaf)#gmPj$P2U+$(|5&Lj3v&p zhx9C&`@QH|AoYK-i$$iY>c|J}Oyv*M!8&?P`u@70aHB4eDRmw?lo7qV=%>^r*f*%C zscW1uqGtqZ9kFHv8bQ;mRUqcMCXCZ+G1+%F)4p&! z?N&39#|jkv{0eLIV`e@G0zmQe{cI?aLdDHYh!W17STs2Rf#q@wX->4rkvTyT6FJ%j zaUp|%QGcA`(dllW3_j}Fscy}AWtBpBXtixQR>N-3sjd|yLK%ol9)6G+^IGn2l@$6M|B9Z#!x)(Q2 zoxSglH{XXBrsgiXY0QO{vFwQPYbTFgQCprHxANxLRUKC~SEfosxT@jeRUOkG{qC=+ zY_0150@VqdK@^tLzJ;FquLf}dIUuBf5Ca0UV)tqqh3s*s5MoUn0;uU2av)+UITI6G zh*M1~12Rw_WS(HpJkemkBwwanHbYtftD8`qpk{eF7Qs8(E3(jJ{#s&~@u}?A#g3)s{W>sCCgzkE0tOgwnji z4<79M(WIdX8dW2*`Y2ybzQkS6fdmB^ltdd*IuxOWT_@eR6K>3fV3iz-XfPB}i2}39 z_<@E<;Fm@MBO@~4mqxO&D0Ltc%sY)TzclKw)ad|5G|H4$bigBxhW#X}pIeKva7EPX zg+zv|KMH%IFgiIwC$@$>r1u`SV`;1gaN!An7SsRk{p3@B0s0wt{+P>w3R z5-3t-K#}^{KFo4_^omS>D$E;ISZYm~93K=|^YdrlbUQtz%D{D_^L}$gvX~RRW$30( z0p+wY3~@bqEeUkl_Ob(Yb!&;8Lf+}4Vrr^cQXtBfB1;f(t*hR-?uj-_?N;quuDtT0 zQQbf99yhIp%@TQDM&_LC{o+@jaX)V?!SMbNh3ko)#G7Sr?BLu@jt-2Tn+$A42|uhFAk;w^$*H$opxs zug^x$*!MIwKcd|m8>U3QFZ4g!rB;!rdR%jgj9)TgwaV3)Yp_#^+W_`Y(SKW_fj=@m~a zDV3G_YHVwXUNMAc&9SJPzqXUuFnI^qvLt6EGi%WBgduS{)Ux`X{&(GGsuOgwMC(`# zLWkjyh*i59=j^Nbt8j-s&t7HUZ(|$tWWmzu@FwQmilX{lZ@pJ_XdR>fJZq2qPr6}_ zRT@cxg0W248Cy1mk51en0EZZ@vj*(5YF>4vtUi?d*&ZJC^*XlBhHG z?WIS)XtLS%|514)8P;g}pfr6<+*UqiTDP9o(&)GSM~Sa<6d3&ny1{TWP1fHT!Ke}B?d(S$4=L%XneP9>y0?20*NpPS`8csj4 z3tWKqdRH3w@PYS5+57DNqdRb#Pt)Gs?d?G;Xruq&E+-F$k_XMVhG-77JqSwWypmGZ z)nc&q{rCQ;NM;f_6lE1rTnz^&7M|bxY%AaJ>~oJcUASv&q5Jt~hrCBb^v{mjuc2G| zHod+d%{p@kt=jqZn?zQd#47kNBC9O~Y3mJNR$BuY0vDeJa`-yG#DuEIBSl1_P8i!{ z4xd?{W4z6p1}1l#4m2}yuX)cE@%(i2C;U(3&Bjm79-&9yF5fP-tJ92)a1-97?nZmGKdP@8Z&-)8ca*pEzgQn*Cs?DZ1{~Xr>BK&y zwf(L^DN_VU2N)xfwR54ImCO0Xd$UB5lVnAKqNoTwN6dYbm|r?HP2Ez7C5UvD(=05S z$i&t@uunl&7ARo=6b|Y8bf{-FF05%BRuqmyBJn6S4d6BhI^*@#S_Yfn6P492CR%v6 zD_$Y4VLenYb&WK)8fDtZV2`&~z3g{~XNsBVW$R=^@eU4N-!U&&|c`ITEC)jCHEhMlM-ky@uy=Ru+6JHVEzExn!KNS?RD$DkUvYjvaRO5!b<`a zp@KZK-5Fp;t(X~0aoFTkE)_Q&Q58e3W2RMCOZ!Y2ug@h8Ct~{(af|L)Jd3Hhy6+ErT}*Fo1iFFMHC125TS!VI`9iRmdeT5eh! zuC;006cgV#s9_7ZNT{eNOs@3A=jEsPV1n010tAVwY0 zfOX|!YzpPw^^8%8ls1TTj1lShJn`L+1GWD+ES8Po{v*4Z^ElI-C&t+Eoh5nOZ%xro zT`Ahv%jeS~^E4N#@$jh4tTNO$PwDR?RA%lBj~Y69T-2@>)WZCi4(BrExsSRF^DeF$ zzG`Y?;pS(p+N$JSvy`vx+kD&FRZG#_Gk@HAN&6Ja%e6!*9wfYMz~AkHG2f{4`^`Ih z(5o_X;0Py5(N2GI6+*y^${6FG|McYB)sBDS7#y<@caOSM6@Aq!AHqq0pk|N+%Cd zi-M7iaTX$?%@L$Mp3(P&9JyosJso&I&cTe6oPp6EXm%6bE{M+~`ny#UjRCXFT}Izx zKw!AW3}cOP(h!*WTSZS9O8{OC;Z^|Kz^$Mka3IF=ZE@3DQbENR4YqqoC997T$8rp0 zZuQX^)@bYF*3o?*vziXyy|9dz{oY`HOo>eP&LOZ`rm;tF^kh1gpk%X7>H=))jN$=Z|o*_{g%p>sBg1qch(H?&B-D)AYA;;fSI}=_AqHTMjNQ z6n5<{6qX+3PA@vRh)@viUi|hV@P8|}tGfziZe(+Ga%Ev{3T19&Z(?c+GdUnIAa7!7 z3Oqb7RC#b^ATLI5ZgfOtb7OL8aCC2S3NJ=)ZgfszZDk-YK|w)5K|w1)a%o{~O;idm zMr>hcb09PdFF|u-Wo~pJIWRRYGB-0cGzu?7WpiU?Zge0mGBGhOIWaUaHVQ9ObY*Q; zAT%{DI5{{pGdKz_Lt$`8Woc(f7)J3Ff%sJivM_@_x*QcGRr z;I44O-Id-|DOHowG++%ewis&A1}ac&5fhD|;RD2=wPH;4lPGF1{t2NG5Mm`wjn6FJ z?%r5TQHxqJfiCeC*L3%@RB*psRp>iOQM+ z3=qgwZipq2i@@bD#-fIy02YT%qAR+hL}jKMx`a-)xlfr)Pzn}R(w(fH0#U3x)=kNs zdWk2_9SPOR=c=Au&Ql^8DvfegXUqZ%N@9j*X1a{`c)wH6Og*|=nBu(H`O5+)6JgE+ zr*=z1zcFz)nxpy7QsA*ZZ#oNy1#4M+p@ojBXepqSgiu4au#e3aNh2OCp6X$D{U6YW zQb3PfoR2m5{WE_j@hf8yQoOJ~j;n(Mve3$yB;N5}VEetp%-*>C@> z8>&2V__d?w)2n8Ow@v=Gv3KCJ>SM3=ZT;%VqNg?-t_%Iz5P$jo62C4FD%Y9?koTLuEB$g_jfO;E&Kf0-pwOh7Pa%zN2J=HN& z{^X6SFOUCns+q_BdOLA-X2;a-{EMxZ`cFps=hoja6H6|80OZ4TB9gY%E$L=Ztsb8wjjHARr{eB5dXd++)xAiz_4+4r8x}jK!08X?6QES!&*!0qnZj(o?l};Fpg2~ znMa{Bf!`6GA^couN^il5!yPYphvIIuk$U!pE^JvG!+aspS+EZwWro5+H8u4O4~PB% D^{*Hk literal 0 HcmV?d00001 diff --git a/tests/verifications/openai_api/fixtures/test_cases/responses.yaml b/tests/verifications/openai_api/fixtures/test_cases/responses.yaml index 1ce25181e..4860715cf 100644 --- a/tests/verifications/openai_api/fixtures/test_cases/responses.yaml +++ b/tests/verifications/openai_api/fixtures/test_cases/responses.yaml @@ -39,7 +39,15 @@ test_response_file_search: input: "How many experts does the Llama 4 Maverick model have?" tools: - type: file_search - # vector_store_ids gets added by the test runner + # vector_store_ids param for file_search tool gets added by the test runner + file_content: "Llama 4 Maverick has 128 experts" + output: "128" + - case_id: "What is the " + input: "How many experts does the Llama 4 Maverick model have?" + tools: + - type: file_search + # vector_store_ids param for file_search toolgets added by the test runner + file_path: "pdfs/llama_stack_and_models.pdf" output: "128" test_response_mcp_tool: diff --git a/tests/verifications/openai_api/test_responses.py b/tests/verifications/openai_api/test_responses.py index f3e306e63..66eada4ba 100644 --- a/tests/verifications/openai_api/test_responses.py +++ b/tests/verifications/openai_api/test_responses.py @@ -5,6 +5,7 @@ # the root directory of this source tree. import json +import os import time import httpx @@ -38,7 +39,7 @@ def _new_vector_store(openai_client, name): return vector_store -def _new_file(openai_client, name, content, tmp_path): +def _upload_file(openai_client, name, file_path): # Ensure we don't reuse an existing file files = openai_client.files.list() for file in files: @@ -46,8 +47,6 @@ def _new_file(openai_client, name, content, tmp_path): openai_client.files.delete(file_id=file.id) # Upload a text file with our document content - file_path = tmp_path / name - file_path.write_text(content) return openai_client.files.create(file=open(file_path, "rb"), purpose="assistants") @@ -291,7 +290,7 @@ def test_response_non_streaming_web_search(request, openai_client, model, provid responses_test_cases["test_response_file_search"]["test_params"]["case"], ids=case_id_generator, ) -def test_response_non_streaming_file_search_simple_text( +def test_response_non_streaming_file_search( request, openai_client, model, provider, verification_config, tmp_path, case ): if isinstance(openai_client, LlamaStackAsLibraryClient): @@ -303,8 +302,17 @@ def test_response_non_streaming_file_search_simple_text( vector_store = _new_vector_store(openai_client, "test_vector_store") - file_content = "Llama 4 Maverick has 128 experts" - file_response = _new_file(openai_client, "test_response_non_streaming_file_search.txt", file_content, tmp_path) + if "file_content" in case: + file_name = "test_response_non_streaming_file_search.txt" + file_path = tmp_path / file_name + file_path.write_text(case["file_content"]) + elif "file_path" in case: + file_path = os.path.join(os.path.dirname(__file__), "fixtures", case["file_path"]) + file_name = os.path.basename(file_path) + else: + raise ValueError(f"No file content or path provided for case {case['case_id']}") + + file_response = _upload_file(openai_client, file_name, file_path) # Attach our file to the vector store file_attach_response = openai_client.vector_stores.files.create( @@ -343,7 +351,7 @@ def test_response_non_streaming_file_search_simple_text( assert response.output[0].status == "completed" assert response.output[0].queries # ensure it's some non-empty list assert response.output[0].results - assert response.output[0].results[0].text == file_content + assert case["output"].lower() in response.output[0].results[0].text.lower() assert response.output[0].results[0].score > 0 # Verify the assistant response that summarizes the results @@ -354,13 +362,8 @@ def test_response_non_streaming_file_search_simple_text( assert case["output"].lower() in response.output_text.lower().strip() -@pytest.mark.parametrize( - "case", - responses_test_cases["test_response_file_search"]["test_params"]["case"], - ids=case_id_generator, -) def test_response_non_streaming_file_search_empty_vector_store( - request, openai_client, model, provider, verification_config, tmp_path, case + request, openai_client, model, provider, verification_config ): if isinstance(openai_client, LlamaStackAsLibraryClient): pytest.skip("Responses API file search is not yet supported in library client.") @@ -371,17 +374,11 @@ def test_response_non_streaming_file_search_empty_vector_store( vector_store = _new_vector_store(openai_client, "test_vector_store") - # Update our tools with the right vector store id - tools = case["tools"] - for tool in tools: - if tool["type"] == "file_search": - tool["vector_store_ids"] = [vector_store.id] - # Create the response request, which should query our vector store response = openai_client.responses.create( model=model, - input=case["input"], - tools=case["tools"], + input="How many experts does the Llama 4 Maverick model have?", + tools=[{"type": "file_search", "vector_store_ids": [vector_store.id]}], stream=False, include=["file_search_call.results"], )