From 5be966dc09c002c178639484f64b38f8104d87a2 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 20 May 2024 17:47:25 -0700
Subject: [PATCH 1/2] feat - add abatch_completion_one_model_multiple_requests

---
 litellm/router.py | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/litellm/router.py b/litellm/router.py
index f022a1f14..d63a36b73 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -662,6 +662,10 @@ class Router:
     async def abatch_completion(
         self, models: List[str], messages: List[Dict[str, str]], **kwargs
     ):
+        """
+        Async Batch Completion - Batch Process 1 request to multiple model_group on litellm.Router
+        Use this for sending the same request to N models
+        """
 
         async def _async_completion_no_exceptions(
             model: str, messages: List[Dict[str, str]], **kwargs
@@ -686,6 +690,43 @@ class Router:
         response = await asyncio.gather(*_tasks)
         return response
 
+    async def abatch_completion_one_model_multiple_requests(
+        self, model: str, messages: List[List[Dict[str, str]]], **kwargs
+    ):
+        """
+        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router
+
+        Use this for sending multiple requests to 1 model
+
+        Args:
+            model (List[str]): model group
+            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
+            **kwargs: additional kwargs
+        """
+
+        async def _async_completion_no_exceptions(
+            model: str, messages: List[Dict[str, str]], **kwargs
+        ):
+            """
+            Wrapper around self.async_completion that catches exceptions and returns them as a result
+            """
+            try:
+                return await self.acompletion(model=model, messages=messages, **kwargs)
+            except Exception as e:
+                return e
+
+        _tasks = []
+        for message_request in messages:
+            # add each task but if the task fails
+            _tasks.append(
+                _async_completion_no_exceptions(
+                    model=model, messages=message_request, **kwargs
+                )
+            )
+
+        response = await asyncio.gather(*_tasks)
+        return response
+
     def image_generation(self, prompt: str, model: str, **kwargs):
         try:
             kwargs["model"] = model

From 92a4df00d4ee27e8e423d55b2b53ee929a6b251d Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 20 May 2024 17:51:08 -0700
Subject: [PATCH 2/2] fix add doc string for
 abatch_completion_one_model_multiple_requests

---
 litellm/router.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/litellm/router.py b/litellm/router.py
index d63a36b73..d678e5912 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -702,6 +702,14 @@ class Router:
             model (List[str]): model group
             messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
             **kwargs: additional kwargs
+        Usage:
+            response = await self.abatch_completion_one_model_multiple_requests(
+                model="gpt-3.5-turbo",
+                messages=[
+                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
+                    [{"role": "user", "content": "hello good mornign"}],
+                ]
+            )
         """
 
         async def _async_completion_no_exceptions(