forked from phoenix/litellm-mirror
cookbook petals
This commit is contained in:
ishaan-jaff
parent
b81f8d2ddd
commit
daf76e900f
1 changed files with 413 additions and 410 deletions
823
cookbook/LiteLLM_Petals.ipynb
vendored
823
cookbook/LiteLLM_Petals.ipynb
vendored
|
|
@ -1,375 +1,14 @@
|
||||||
{
|
{
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 0,
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"provenance": [],
|
|
||||||
"gpuType": "T4"
|
|
||||||
},
|
|
||||||
"kernelspec": {
|
|
||||||
"name": "python3",
|
|
||||||
"display_name": "Python 3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"name": "python"
|
|
||||||
},
|
|
||||||
"accelerator": "GPU",
|
|
||||||
"widgets": {
|
|
||||||
"application/vnd.jupyter.widget-state+json": {
|
|
||||||
"2fec5cc400424671a3d517327117d18a": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "HBoxModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_dom_classes": [],
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "HBoxModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/controls",
|
|
||||||
"_view_module_version": "1.5.0",
|
|
||||||
"_view_name": "HBoxView",
|
|
||||||
"box_style": "",
|
|
||||||
"children": [
|
|
||||||
"IPY_MODEL_3687c76fe84d464baaf35366b21e83b3",
|
|
||||||
"IPY_MODEL_c29d4460dbaa441cae110b58e0014151",
|
|
||||||
"IPY_MODEL_6560449a38bf4a7bacd97ccaacf01c4c"
|
|
||||||
],
|
|
||||||
"layout": "IPY_MODEL_5fbd6ae281984d28ba59ebfd0279eda7"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"3687c76fe84d464baaf35366b21e83b3": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "HTMLModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_dom_classes": [],
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "HTMLModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/controls",
|
|
||||||
"_view_module_version": "1.5.0",
|
|
||||||
"_view_name": "HTMLView",
|
|
||||||
"description": "",
|
|
||||||
"description_tooltip": null,
|
|
||||||
"layout": "IPY_MODEL_323e30e275434aeea241163e5f1f9031",
|
|
||||||
"placeholder": "",
|
|
||||||
"style": "IPY_MODEL_48f4adec51c94f9da6e4c4564daeff84",
|
|
||||||
"value": "Loading checkpoint shards: 100%"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"c29d4460dbaa441cae110b58e0014151": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "FloatProgressModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_dom_classes": [],
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "FloatProgressModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/controls",
|
|
||||||
"_view_module_version": "1.5.0",
|
|
||||||
"_view_name": "ProgressView",
|
|
||||||
"bar_style": "success",
|
|
||||||
"description": "",
|
|
||||||
"description_tooltip": null,
|
|
||||||
"layout": "IPY_MODEL_2a672981a44b4a7fb30674f97f4c10c6",
|
|
||||||
"max": 2,
|
|
||||||
"min": 0,
|
|
||||||
"orientation": "horizontal",
|
|
||||||
"style": "IPY_MODEL_d75ae8d22ea74840b4c80c8f386384c4",
|
|
||||||
"value": 2
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"6560449a38bf4a7bacd97ccaacf01c4c": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "HTMLModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_dom_classes": [],
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "HTMLModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/controls",
|
|
||||||
"_view_module_version": "1.5.0",
|
|
||||||
"_view_name": "HTMLView",
|
|
||||||
"description": "",
|
|
||||||
"description_tooltip": null,
|
|
||||||
"layout": "IPY_MODEL_54c06312ecff4e7588665e8b0cb7118b",
|
|
||||||
"placeholder": "",
|
|
||||||
"style": "IPY_MODEL_300078a9d1a6483fba81a4be63793ff7",
|
|
||||||
"value": " 2/2 [00:00<00:00, 2.36it/s]"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"5fbd6ae281984d28ba59ebfd0279eda7": {
|
|
||||||
"model_module": "@jupyter-widgets/base",
|
|
||||||
"model_name": "LayoutModel",
|
|
||||||
"model_module_version": "1.2.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/base",
|
|
||||||
"_model_module_version": "1.2.0",
|
|
||||||
"_model_name": "LayoutModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "LayoutView",
|
|
||||||
"align_content": null,
|
|
||||||
"align_items": null,
|
|
||||||
"align_self": null,
|
|
||||||
"border": null,
|
|
||||||
"bottom": null,
|
|
||||||
"display": null,
|
|
||||||
"flex": null,
|
|
||||||
"flex_flow": null,
|
|
||||||
"grid_area": null,
|
|
||||||
"grid_auto_columns": null,
|
|
||||||
"grid_auto_flow": null,
|
|
||||||
"grid_auto_rows": null,
|
|
||||||
"grid_column": null,
|
|
||||||
"grid_gap": null,
|
|
||||||
"grid_row": null,
|
|
||||||
"grid_template_areas": null,
|
|
||||||
"grid_template_columns": null,
|
|
||||||
"grid_template_rows": null,
|
|
||||||
"height": null,
|
|
||||||
"justify_content": null,
|
|
||||||
"justify_items": null,
|
|
||||||
"left": null,
|
|
||||||
"margin": null,
|
|
||||||
"max_height": null,
|
|
||||||
"max_width": null,
|
|
||||||
"min_height": null,
|
|
||||||
"min_width": null,
|
|
||||||
"object_fit": null,
|
|
||||||
"object_position": null,
|
|
||||||
"order": null,
|
|
||||||
"overflow": null,
|
|
||||||
"overflow_x": null,
|
|
||||||
"overflow_y": null,
|
|
||||||
"padding": null,
|
|
||||||
"right": null,
|
|
||||||
"top": null,
|
|
||||||
"visibility": null,
|
|
||||||
"width": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"323e30e275434aeea241163e5f1f9031": {
|
|
||||||
"model_module": "@jupyter-widgets/base",
|
|
||||||
"model_name": "LayoutModel",
|
|
||||||
"model_module_version": "1.2.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/base",
|
|
||||||
"_model_module_version": "1.2.0",
|
|
||||||
"_model_name": "LayoutModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "LayoutView",
|
|
||||||
"align_content": null,
|
|
||||||
"align_items": null,
|
|
||||||
"align_self": null,
|
|
||||||
"border": null,
|
|
||||||
"bottom": null,
|
|
||||||
"display": null,
|
|
||||||
"flex": null,
|
|
||||||
"flex_flow": null,
|
|
||||||
"grid_area": null,
|
|
||||||
"grid_auto_columns": null,
|
|
||||||
"grid_auto_flow": null,
|
|
||||||
"grid_auto_rows": null,
|
|
||||||
"grid_column": null,
|
|
||||||
"grid_gap": null,
|
|
||||||
"grid_row": null,
|
|
||||||
"grid_template_areas": null,
|
|
||||||
"grid_template_columns": null,
|
|
||||||
"grid_template_rows": null,
|
|
||||||
"height": null,
|
|
||||||
"justify_content": null,
|
|
||||||
"justify_items": null,
|
|
||||||
"left": null,
|
|
||||||
"margin": null,
|
|
||||||
"max_height": null,
|
|
||||||
"max_width": null,
|
|
||||||
"min_height": null,
|
|
||||||
"min_width": null,
|
|
||||||
"object_fit": null,
|
|
||||||
"object_position": null,
|
|
||||||
"order": null,
|
|
||||||
"overflow": null,
|
|
||||||
"overflow_x": null,
|
|
||||||
"overflow_y": null,
|
|
||||||
"padding": null,
|
|
||||||
"right": null,
|
|
||||||
"top": null,
|
|
||||||
"visibility": null,
|
|
||||||
"width": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"48f4adec51c94f9da6e4c4564daeff84": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "DescriptionStyleModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "DescriptionStyleModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "StyleView",
|
|
||||||
"description_width": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"2a672981a44b4a7fb30674f97f4c10c6": {
|
|
||||||
"model_module": "@jupyter-widgets/base",
|
|
||||||
"model_name": "LayoutModel",
|
|
||||||
"model_module_version": "1.2.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/base",
|
|
||||||
"_model_module_version": "1.2.0",
|
|
||||||
"_model_name": "LayoutModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "LayoutView",
|
|
||||||
"align_content": null,
|
|
||||||
"align_items": null,
|
|
||||||
"align_self": null,
|
|
||||||
"border": null,
|
|
||||||
"bottom": null,
|
|
||||||
"display": null,
|
|
||||||
"flex": null,
|
|
||||||
"flex_flow": null,
|
|
||||||
"grid_area": null,
|
|
||||||
"grid_auto_columns": null,
|
|
||||||
"grid_auto_flow": null,
|
|
||||||
"grid_auto_rows": null,
|
|
||||||
"grid_column": null,
|
|
||||||
"grid_gap": null,
|
|
||||||
"grid_row": null,
|
|
||||||
"grid_template_areas": null,
|
|
||||||
"grid_template_columns": null,
|
|
||||||
"grid_template_rows": null,
|
|
||||||
"height": null,
|
|
||||||
"justify_content": null,
|
|
||||||
"justify_items": null,
|
|
||||||
"left": null,
|
|
||||||
"margin": null,
|
|
||||||
"max_height": null,
|
|
||||||
"max_width": null,
|
|
||||||
"min_height": null,
|
|
||||||
"min_width": null,
|
|
||||||
"object_fit": null,
|
|
||||||
"object_position": null,
|
|
||||||
"order": null,
|
|
||||||
"overflow": null,
|
|
||||||
"overflow_x": null,
|
|
||||||
"overflow_y": null,
|
|
||||||
"padding": null,
|
|
||||||
"right": null,
|
|
||||||
"top": null,
|
|
||||||
"visibility": null,
|
|
||||||
"width": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"d75ae8d22ea74840b4c80c8f386384c4": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "ProgressStyleModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "ProgressStyleModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "StyleView",
|
|
||||||
"bar_color": null,
|
|
||||||
"description_width": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"54c06312ecff4e7588665e8b0cb7118b": {
|
|
||||||
"model_module": "@jupyter-widgets/base",
|
|
||||||
"model_name": "LayoutModel",
|
|
||||||
"model_module_version": "1.2.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/base",
|
|
||||||
"_model_module_version": "1.2.0",
|
|
||||||
"_model_name": "LayoutModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "LayoutView",
|
|
||||||
"align_content": null,
|
|
||||||
"align_items": null,
|
|
||||||
"align_self": null,
|
|
||||||
"border": null,
|
|
||||||
"bottom": null,
|
|
||||||
"display": null,
|
|
||||||
"flex": null,
|
|
||||||
"flex_flow": null,
|
|
||||||
"grid_area": null,
|
|
||||||
"grid_auto_columns": null,
|
|
||||||
"grid_auto_flow": null,
|
|
||||||
"grid_auto_rows": null,
|
|
||||||
"grid_column": null,
|
|
||||||
"grid_gap": null,
|
|
||||||
"grid_row": null,
|
|
||||||
"grid_template_areas": null,
|
|
||||||
"grid_template_columns": null,
|
|
||||||
"grid_template_rows": null,
|
|
||||||
"height": null,
|
|
||||||
"justify_content": null,
|
|
||||||
"justify_items": null,
|
|
||||||
"left": null,
|
|
||||||
"margin": null,
|
|
||||||
"max_height": null,
|
|
||||||
"max_width": null,
|
|
||||||
"min_height": null,
|
|
||||||
"min_width": null,
|
|
||||||
"object_fit": null,
|
|
||||||
"object_position": null,
|
|
||||||
"order": null,
|
|
||||||
"overflow": null,
|
|
||||||
"overflow_x": null,
|
|
||||||
"overflow_y": null,
|
|
||||||
"padding": null,
|
|
||||||
"right": null,
|
|
||||||
"top": null,
|
|
||||||
"visibility": null,
|
|
||||||
"width": null
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"300078a9d1a6483fba81a4be63793ff7": {
|
|
||||||
"model_module": "@jupyter-widgets/controls",
|
|
||||||
"model_name": "DescriptionStyleModel",
|
|
||||||
"model_module_version": "1.5.0",
|
|
||||||
"state": {
|
|
||||||
"_model_module": "@jupyter-widgets/controls",
|
|
||||||
"_model_module_version": "1.5.0",
|
|
||||||
"_model_name": "DescriptionStyleModel",
|
|
||||||
"_view_count": null,
|
|
||||||
"_view_module": "@jupyter-widgets/base",
|
|
||||||
"_view_module_version": "1.2.0",
|
|
||||||
"_view_name": "StyleView",
|
|
||||||
"description_width": ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
|
"attachments": {},
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"source": [
|
|
||||||
"# Using LiteLLM with Petals"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "dwGtLi_tvM6N"
|
"id": "dwGtLi_tvM6N"
|
||||||
}
|
},
|
||||||
|
"source": [
|
||||||
|
"# Using LiteLLM with Petals"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
|
|
@ -379,39 +18,34 @@
|
||||||
},
|
},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"!pip install litellm # 0.1.702 and upwards"
|
"!pip install litellm # 0.1.715 and upwards"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"source": [
|
"execution_count": null,
|
||||||
"# install petals\n",
|
|
||||||
"!pip install git+https://github.com/bigscience-workshop/petals"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "5Id2QKwOEH8X"
|
"id": "5Id2QKwOEH8X"
|
||||||
},
|
},
|
||||||
"execution_count": null,
|
"outputs": [],
|
||||||
"outputs": []
|
"source": [
|
||||||
|
"# install petals\n",
|
||||||
|
"!pip install git+https://github.com/bigscience-workshop/petals"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"attachments": {},
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"source": [
|
|
||||||
"## petals-team/StableBeluga2"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "k42fldw3veSN"
|
"id": "k42fldw3veSN"
|
||||||
}
|
},
|
||||||
|
"source": [
|
||||||
|
"## petals-team/StableBeluga2"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"source": [
|
"execution_count": 3,
|
||||||
"from litellm import completion\n",
|
|
||||||
"\n",
|
|
||||||
"response = completion(model=\"petals/petals-team/StableBeluga2\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], max_tokens=50)\n",
|
|
||||||
"\n",
|
|
||||||
"print(response)"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"colab": {
|
"colab": {
|
||||||
"base_uri": "https://localhost:8080/"
|
"base_uri": "https://localhost:8080/"
|
||||||
|
|
@ -419,11 +53,10 @@
|
||||||
"id": "tIHcEHdSDqju",
|
"id": "tIHcEHdSDqju",
|
||||||
"outputId": "485dbf54-395c-433a-bbf4-8eb70a9fa624"
|
"outputId": "485dbf54-395c-433a-bbf4-8eb70a9fa624"
|
||||||
},
|
},
|
||||||
"execution_count": 3,
|
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
|
"You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
|
||||||
"Sep 19 18:39:50.634 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
|
"Sep 19 18:39:50.634 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
|
||||||
|
|
@ -432,8 +65,8 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stdout",
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"{\n",
|
"{\n",
|
||||||
" \"object\": \"chat.completion\",\n",
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
|
@ -459,24 +92,28 @@
|
||||||
"}\n"
|
"}\n"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from litellm import completion\n",
|
||||||
|
"\n",
|
||||||
|
"response = completion(model=\"petals/petals-team/StableBeluga2\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], max_tokens=50)\n",
|
||||||
|
"\n",
|
||||||
|
"print(response)"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"attachments": {},
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"source": [
|
|
||||||
"## huggyllama/llama-65b"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "J8DubRnHvh_j"
|
"id": "J8DubRnHvh_j"
|
||||||
}
|
},
|
||||||
|
"source": [
|
||||||
|
"## huggyllama/llama-65b"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"cell_type": "code",
|
"cell_type": "code",
|
||||||
"source": [
|
"execution_count": 4,
|
||||||
"response = completion(model=\"petals/huggyllama/llama-65b\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], temperature=0.2, max_tokens=10)\n",
|
|
||||||
"\n",
|
|
||||||
"print(response)"
|
|
||||||
],
|
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"colab": {
|
"colab": {
|
||||||
"base_uri": "https://localhost:8080/",
|
"base_uri": "https://localhost:8080/",
|
||||||
|
|
@ -498,33 +135,32 @@
|
||||||
"id": "IlTCJwDsNvgF",
|
"id": "IlTCJwDsNvgF",
|
||||||
"outputId": "2e84d125-d982-48ed-8a92-6ca438a50d0c"
|
"outputId": "2e84d125-d982-48ed-8a92-6ca438a50d0c"
|
||||||
},
|
},
|
||||||
"execution_count": 4,
|
|
||||||
"outputs": [
|
"outputs": [
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"Sep 19 18:41:37.912 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
|
"Sep 19 18:41:37.912 [\u001b[1m\u001b[34mINFO\u001b[0m] Make sure you follow the LLaMA's terms of use: https://bit.ly/llama2-license for LLaMA 2, https://bit.ly/llama-license for LLaMA 1\n",
|
||||||
"Sep 19 18:41:37.914 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: llama-65b-hf\n"
|
"Sep 19 18:41:37.914 [\u001b[1m\u001b[34mINFO\u001b[0m] Using DHT prefix: llama-65b-hf\n"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"output_type": "display_data",
|
|
||||||
"data": {
|
"data": {
|
||||||
|
"application/vnd.jupyter.widget-view+json": {
|
||||||
|
"model_id": "2fec5cc400424671a3d517327117d18a",
|
||||||
|
"version_major": 2,
|
||||||
|
"version_minor": 0
|
||||||
|
},
|
||||||
"text/plain": [
|
"text/plain": [
|
||||||
"Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]"
|
"Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]"
|
||||||
],
|
]
|
||||||
"application/vnd.jupyter.widget-view+json": {
|
|
||||||
"version_major": 2,
|
|
||||||
"version_minor": 0,
|
|
||||||
"model_id": "2fec5cc400424671a3d517327117d18a"
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"metadata": {}
|
"metadata": {},
|
||||||
|
"output_type": "display_data"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stderr",
|
"name": "stderr",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.2` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
|
"/usr/local/lib/python3.10/dist-packages/transformers/generation/configuration_utils.py:362: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.2` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
|
||||||
" warnings.warn(\n",
|
" warnings.warn(\n",
|
||||||
|
|
@ -532,8 +168,8 @@
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stdout",
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
"text": [
|
"text": [
|
||||||
"{\n",
|
"{\n",
|
||||||
" \"object\": \"chat.completion\",\n",
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
|
@ -559,7 +195,374 @@
|
||||||
"}\n"
|
"}\n"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"response = completion(model=\"petals/huggyllama/llama-65b\", messages=[{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}], temperature=0.2, max_tokens=10)\n",
|
||||||
|
"\n",
|
||||||
|
"print(response)"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
}
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"colab": {
|
||||||
|
"gpuType": "T4",
|
||||||
|
"provenance": []
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"name": "python"
|
||||||
|
},
|
||||||
|
"widgets": {
|
||||||
|
"application/vnd.jupyter.widget-state+json": {
|
||||||
|
"2a672981a44b4a7fb30674f97f4c10c6": {
|
||||||
|
"model_module": "@jupyter-widgets/base",
|
||||||
|
"model_module_version": "1.2.0",
|
||||||
|
"model_name": "LayoutModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/base",
|
||||||
|
"_model_module_version": "1.2.0",
|
||||||
|
"_model_name": "LayoutModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "LayoutView",
|
||||||
|
"align_content": null,
|
||||||
|
"align_items": null,
|
||||||
|
"align_self": null,
|
||||||
|
"border": null,
|
||||||
|
"bottom": null,
|
||||||
|
"display": null,
|
||||||
|
"flex": null,
|
||||||
|
"flex_flow": null,
|
||||||
|
"grid_area": null,
|
||||||
|
"grid_auto_columns": null,
|
||||||
|
"grid_auto_flow": null,
|
||||||
|
"grid_auto_rows": null,
|
||||||
|
"grid_column": null,
|
||||||
|
"grid_gap": null,
|
||||||
|
"grid_row": null,
|
||||||
|
"grid_template_areas": null,
|
||||||
|
"grid_template_columns": null,
|
||||||
|
"grid_template_rows": null,
|
||||||
|
"height": null,
|
||||||
|
"justify_content": null,
|
||||||
|
"justify_items": null,
|
||||||
|
"left": null,
|
||||||
|
"margin": null,
|
||||||
|
"max_height": null,
|
||||||
|
"max_width": null,
|
||||||
|
"min_height": null,
|
||||||
|
"min_width": null,
|
||||||
|
"object_fit": null,
|
||||||
|
"object_position": null,
|
||||||
|
"order": null,
|
||||||
|
"overflow": null,
|
||||||
|
"overflow_x": null,
|
||||||
|
"overflow_y": null,
|
||||||
|
"padding": null,
|
||||||
|
"right": null,
|
||||||
|
"top": null,
|
||||||
|
"visibility": null,
|
||||||
|
"width": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"2fec5cc400424671a3d517327117d18a": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "HBoxModel",
|
||||||
|
"state": {
|
||||||
|
"_dom_classes": [],
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "HBoxModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/controls",
|
||||||
|
"_view_module_version": "1.5.0",
|
||||||
|
"_view_name": "HBoxView",
|
||||||
|
"box_style": "",
|
||||||
|
"children": [
|
||||||
|
"IPY_MODEL_3687c76fe84d464baaf35366b21e83b3",
|
||||||
|
"IPY_MODEL_c29d4460dbaa441cae110b58e0014151",
|
||||||
|
"IPY_MODEL_6560449a38bf4a7bacd97ccaacf01c4c"
|
||||||
|
],
|
||||||
|
"layout": "IPY_MODEL_5fbd6ae281984d28ba59ebfd0279eda7"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"300078a9d1a6483fba81a4be63793ff7": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "DescriptionStyleModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "DescriptionStyleModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "StyleView",
|
||||||
|
"description_width": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"323e30e275434aeea241163e5f1f9031": {
|
||||||
|
"model_module": "@jupyter-widgets/base",
|
||||||
|
"model_module_version": "1.2.0",
|
||||||
|
"model_name": "LayoutModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/base",
|
||||||
|
"_model_module_version": "1.2.0",
|
||||||
|
"_model_name": "LayoutModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "LayoutView",
|
||||||
|
"align_content": null,
|
||||||
|
"align_items": null,
|
||||||
|
"align_self": null,
|
||||||
|
"border": null,
|
||||||
|
"bottom": null,
|
||||||
|
"display": null,
|
||||||
|
"flex": null,
|
||||||
|
"flex_flow": null,
|
||||||
|
"grid_area": null,
|
||||||
|
"grid_auto_columns": null,
|
||||||
|
"grid_auto_flow": null,
|
||||||
|
"grid_auto_rows": null,
|
||||||
|
"grid_column": null,
|
||||||
|
"grid_gap": null,
|
||||||
|
"grid_row": null,
|
||||||
|
"grid_template_areas": null,
|
||||||
|
"grid_template_columns": null,
|
||||||
|
"grid_template_rows": null,
|
||||||
|
"height": null,
|
||||||
|
"justify_content": null,
|
||||||
|
"justify_items": null,
|
||||||
|
"left": null,
|
||||||
|
"margin": null,
|
||||||
|
"max_height": null,
|
||||||
|
"max_width": null,
|
||||||
|
"min_height": null,
|
||||||
|
"min_width": null,
|
||||||
|
"object_fit": null,
|
||||||
|
"object_position": null,
|
||||||
|
"order": null,
|
||||||
|
"overflow": null,
|
||||||
|
"overflow_x": null,
|
||||||
|
"overflow_y": null,
|
||||||
|
"padding": null,
|
||||||
|
"right": null,
|
||||||
|
"top": null,
|
||||||
|
"visibility": null,
|
||||||
|
"width": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"3687c76fe84d464baaf35366b21e83b3": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "HTMLModel",
|
||||||
|
"state": {
|
||||||
|
"_dom_classes": [],
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "HTMLModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/controls",
|
||||||
|
"_view_module_version": "1.5.0",
|
||||||
|
"_view_name": "HTMLView",
|
||||||
|
"description": "",
|
||||||
|
"description_tooltip": null,
|
||||||
|
"layout": "IPY_MODEL_323e30e275434aeea241163e5f1f9031",
|
||||||
|
"placeholder": "",
|
||||||
|
"style": "IPY_MODEL_48f4adec51c94f9da6e4c4564daeff84",
|
||||||
|
"value": "Loading checkpoint shards: 100%"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"48f4adec51c94f9da6e4c4564daeff84": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "DescriptionStyleModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "DescriptionStyleModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "StyleView",
|
||||||
|
"description_width": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"54c06312ecff4e7588665e8b0cb7118b": {
|
||||||
|
"model_module": "@jupyter-widgets/base",
|
||||||
|
"model_module_version": "1.2.0",
|
||||||
|
"model_name": "LayoutModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/base",
|
||||||
|
"_model_module_version": "1.2.0",
|
||||||
|
"_model_name": "LayoutModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "LayoutView",
|
||||||
|
"align_content": null,
|
||||||
|
"align_items": null,
|
||||||
|
"align_self": null,
|
||||||
|
"border": null,
|
||||||
|
"bottom": null,
|
||||||
|
"display": null,
|
||||||
|
"flex": null,
|
||||||
|
"flex_flow": null,
|
||||||
|
"grid_area": null,
|
||||||
|
"grid_auto_columns": null,
|
||||||
|
"grid_auto_flow": null,
|
||||||
|
"grid_auto_rows": null,
|
||||||
|
"grid_column": null,
|
||||||
|
"grid_gap": null,
|
||||||
|
"grid_row": null,
|
||||||
|
"grid_template_areas": null,
|
||||||
|
"grid_template_columns": null,
|
||||||
|
"grid_template_rows": null,
|
||||||
|
"height": null,
|
||||||
|
"justify_content": null,
|
||||||
|
"justify_items": null,
|
||||||
|
"left": null,
|
||||||
|
"margin": null,
|
||||||
|
"max_height": null,
|
||||||
|
"max_width": null,
|
||||||
|
"min_height": null,
|
||||||
|
"min_width": null,
|
||||||
|
"object_fit": null,
|
||||||
|
"object_position": null,
|
||||||
|
"order": null,
|
||||||
|
"overflow": null,
|
||||||
|
"overflow_x": null,
|
||||||
|
"overflow_y": null,
|
||||||
|
"padding": null,
|
||||||
|
"right": null,
|
||||||
|
"top": null,
|
||||||
|
"visibility": null,
|
||||||
|
"width": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"5fbd6ae281984d28ba59ebfd0279eda7": {
|
||||||
|
"model_module": "@jupyter-widgets/base",
|
||||||
|
"model_module_version": "1.2.0",
|
||||||
|
"model_name": "LayoutModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/base",
|
||||||
|
"_model_module_version": "1.2.0",
|
||||||
|
"_model_name": "LayoutModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "LayoutView",
|
||||||
|
"align_content": null,
|
||||||
|
"align_items": null,
|
||||||
|
"align_self": null,
|
||||||
|
"border": null,
|
||||||
|
"bottom": null,
|
||||||
|
"display": null,
|
||||||
|
"flex": null,
|
||||||
|
"flex_flow": null,
|
||||||
|
"grid_area": null,
|
||||||
|
"grid_auto_columns": null,
|
||||||
|
"grid_auto_flow": null,
|
||||||
|
"grid_auto_rows": null,
|
||||||
|
"grid_column": null,
|
||||||
|
"grid_gap": null,
|
||||||
|
"grid_row": null,
|
||||||
|
"grid_template_areas": null,
|
||||||
|
"grid_template_columns": null,
|
||||||
|
"grid_template_rows": null,
|
||||||
|
"height": null,
|
||||||
|
"justify_content": null,
|
||||||
|
"justify_items": null,
|
||||||
|
"left": null,
|
||||||
|
"margin": null,
|
||||||
|
"max_height": null,
|
||||||
|
"max_width": null,
|
||||||
|
"min_height": null,
|
||||||
|
"min_width": null,
|
||||||
|
"object_fit": null,
|
||||||
|
"object_position": null,
|
||||||
|
"order": null,
|
||||||
|
"overflow": null,
|
||||||
|
"overflow_x": null,
|
||||||
|
"overflow_y": null,
|
||||||
|
"padding": null,
|
||||||
|
"right": null,
|
||||||
|
"top": null,
|
||||||
|
"visibility": null,
|
||||||
|
"width": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"6560449a38bf4a7bacd97ccaacf01c4c": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "HTMLModel",
|
||||||
|
"state": {
|
||||||
|
"_dom_classes": [],
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "HTMLModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/controls",
|
||||||
|
"_view_module_version": "1.5.0",
|
||||||
|
"_view_name": "HTMLView",
|
||||||
|
"description": "",
|
||||||
|
"description_tooltip": null,
|
||||||
|
"layout": "IPY_MODEL_54c06312ecff4e7588665e8b0cb7118b",
|
||||||
|
"placeholder": "",
|
||||||
|
"style": "IPY_MODEL_300078a9d1a6483fba81a4be63793ff7",
|
||||||
|
"value": " 2/2 [00:00<00:00, 2.36it/s]"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"c29d4460dbaa441cae110b58e0014151": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "FloatProgressModel",
|
||||||
|
"state": {
|
||||||
|
"_dom_classes": [],
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "FloatProgressModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/controls",
|
||||||
|
"_view_module_version": "1.5.0",
|
||||||
|
"_view_name": "ProgressView",
|
||||||
|
"bar_style": "success",
|
||||||
|
"description": "",
|
||||||
|
"description_tooltip": null,
|
||||||
|
"layout": "IPY_MODEL_2a672981a44b4a7fb30674f97f4c10c6",
|
||||||
|
"max": 2,
|
||||||
|
"min": 0,
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"style": "IPY_MODEL_d75ae8d22ea74840b4c80c8f386384c4",
|
||||||
|
"value": 2
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"d75ae8d22ea74840b4c80c8f386384c4": {
|
||||||
|
"model_module": "@jupyter-widgets/controls",
|
||||||
|
"model_module_version": "1.5.0",
|
||||||
|
"model_name": "ProgressStyleModel",
|
||||||
|
"state": {
|
||||||
|
"_model_module": "@jupyter-widgets/controls",
|
||||||
|
"_model_module_version": "1.5.0",
|
||||||
|
"_model_name": "ProgressStyleModel",
|
||||||
|
"_view_count": null,
|
||||||
|
"_view_module": "@jupyter-widgets/base",
|
||||||
|
"_view_module_version": "1.2.0",
|
||||||
|
"_view_name": "StyleView",
|
||||||
|
"bar_color": null,
|
||||||
|
"description_width": ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 0
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue