phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

feat - allow sending tags on vertex pass through requests (#6876)

Browse source 
* feat - allow tagging vertex JS SDK request

* add unit testing for passing headers for pass through endpoints

* fix allow using vertex_ai as the primary way for pass through vertex endpoints

* docs on vertex js pass tags

* add e2e test for vertex pass through with spend tags

* add e2e tests for streaming vertex JS with tags

* fix vertex ai testing
This commit is contained in:
Ishaan Jaff 2024-11-25 12:12:09 -08:00 • committed by GitHub
parent c73ce95c01
commit f77bf49772
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 548 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

43
tests/pass_through_tests/test_local_vertex.js
View file

@ -1,31 +1,22 @@
const { VertexAI, RequestOptions } = require('@google-cloud/vertexai');
// Import fetch if the SDK uses it
const originalFetch = global.fetch || require('node-fetch');
// Monkey-patch the fetch used internally
global.fetch = async function patchedFetch(url, options) {
// Modify the URL to use HTTP instead of HTTPS
if (url.startsWith('https://localhost:4000')) {
url = url.replace('https://', 'http://');
}
console.log('Patched fetch sending request to:', url);
return originalFetch(url, options);
};
const vertexAI = new VertexAI({
project: 'adroit-crow-413218',
location: 'us-central1',
apiEndpoint: "localhost:4000/vertex-ai"
apiEndpoint: "127.0.0.1:4000/vertex-ai"
});
// Create customHeaders using Headers
const customHeaders = new Headers({
"X-Litellm-Api-Key": "sk-1234",
tags: "vertexjs,test-2"
});
// Use customHeaders in RequestOptions
const requestOptions = {
customHeaders: new Headers({
"x-litellm-api-key": "sk-1234"
})
customHeaders: customHeaders,
};
const generativeModel = vertexAI.getGenerativeModel(
@ -33,7 +24,7 @@ const generativeModel = vertexAI.getGenerativeModel(
requestOptions
);
async function streamingResponse() {
async function testModel() {
try {
const request = {
contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}],
@ -49,20 +40,4 @@ async function streamingResponse() {
}
}
async function nonStreamingResponse() {
try {
const request = {
contents: [{role: 'user', parts: [{text: 'How are you doing today tell me your name?'}]}],
};
const response = await generativeModel.generateContent(request);
console.log('non streaming response: ', JSON.stringify(response));
} catch (error) {
console.error('Error:', error);
}
}
streamingResponse();
nonStreamingResponse();
testModel();