liteLLM как единый слой для вызова LLM со стримингом

Title:

URL Source: https://raw.githubusercontent.com/BerriAI/litellm/litellm_internal_staging/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb

Markdown Content: { "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "ZwuaylskLxFu", "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting litellm==0.1.363\n", " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n", "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n", "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n", "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n", "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n", "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n", "Installing collected packages: litellm\n", " Attempting uninstall: litellm\n", " Found existing installation: litellm 0.1.362\n", " Uninstalling litellm-0.1.362:\n", " Successfully uninstalled litellm-0.1.362\n", "Successfully installed litellm-0.1.363\n" ] } ], "source": [ "!pip install litellm==\"0.1.363\"" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "id": "W216G__XL19Q" }, "outputs": [], "source": [ "# @title Import litellm & Set env variables\n", "import litellm\n", "import os\n", "\n", "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "ff1lKwUMMLJj", "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "\n", " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", "\n", "\n", " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" ] } ], "source": [ "# @title Request Claude Instant-1 and Claude-2\n", "messages = [\n", " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", " {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n", " ]\n", "\n", "result = litellm.completion('claude-instant-1', messages)\n", "print(\"\\n\\n Result from claude-instant-1\", result)\n", "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n", "print(\"\\n\\n Result from claude-2\", result)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "06hWKnNQMrV-", "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Here\n", "'s\n", " a\n", " quick\n", " overview\n", " of\n", " how\n", " a\n", " court\n", " case\n", " can\n", " reach\n", " the\n", " U\n", ".\n", "S\n", ".\n", " Supreme\n", " Court\n", ":\n", "\n", "\n", "-\n", " The\n", " case\n", " must\n", " first\n", " be\n", " heard\n", " in\n", " a\n", " lower\n", " trial\n", " court\n", " (\n", "either\n", " a\n", " state\n", " court\n", " or\n", " federal\n", " district\n", " court\n", ").\n", " The\n", " trial\n", " court\n", " makes\n", " initial\n", " r\n", "ulings\n", " and\n", " produces\n", " a\n", " record\n", " of\n", " the\n", " case\n", ".\n", "\n", "\n", "-\n", " The\n", " losing\n", " party\n", " can\n", " appeal\n", " the\n", " decision\n", " to\n", " an\n", " appeals\n", " court\n", " (\n", "a\n", " state\n", " appeals\n", " court\n", " for\n", " state\n", " cases\n", ",\n", " or\n", " a\n", " federal\n", " circuit\n", " court\n", " for\n", " federal\n", " cases\n", ").\n", " The\n", " appeals\n", " court\n", " reviews\n", " the\n", " trial\n", " court\n", "'s\n", " r\n", "ulings\n", " and\n", " can\n", " affirm\n", ",\n", " reverse\n", ",\n", " or\n", " modify\n", " the\n", " decision\n", ".\n", "\n", "\n", "-\n", " If\n", " a\n", " party\n", " is\n", " still\n", " unsat\n", "isf\n", "ied\n", " after\n", " the\n", " appeals\n", " court\n", " rules\n", ",\n", " they\n", " can\n", " petition\n", " the\n", " Supreme\n", " Court\n", " to\n", " hear\n", " the\n", " case\n", " through\n", " a\n", " writ\n", " of\n", " cert\n", "ior\n", "ari\n", ".\n", " \n", "\n", "\n", "-\n", " The\n", " Supreme\n", " Court\n", " gets\n", " thousands\n", " of\n", " cert\n", " petitions\n", " every\n", " year\n", " but\n", " usually\n", " only\n", " agrees\n", " to\n", " hear\n", " about\n", " 100\n", "-\n", "150\n", " of\n", " cases\n", " that\n", " have\n", " significant\n", " national\n", " importance\n", " or\n", " where\n", " lower\n", " courts\n", " disagree\n", " on\n", " federal\n", " law\n", ".\n", " \n", "\n", "\n", "-\n", " If\n", " 4\n", " out\n", " of\n", " the\n", " 9\n", " Just\n", "ices\n", " vote\n", " to\n", " grant\n", " cert\n", " (\n", "agree\n", " to\n", " hear\n", " the\n", " case\n", "),\n", " it\n", " goes\n", " on\n", " the\n", " Supreme\n", " Court\n", "'s\n", " do\n", "cket\n", " for\n", " arguments\n", ".\n", "\n", "\n", "-\n", " The\n", " Supreme\n", " Court\n", " then\n", " hears\n", " oral\n", " arguments\n", ",\n", " considers\n", " written\n", " brief\n", "s\n", ",\n", " examines\n", " the\n", " lower\n", " court\n", " records\n", ",\n", " and\n", " issues\n", " a\n", " final\n", " ruling\n", " on\n", " the\n", " case\n", ",\n", " which\n", " serves\n", " as\n", " binding\n", " precedent\n" ] } ], "source": [ "# @title Streaming Example: Request Claude-2\n", "messages = [\n", " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", " ]\n", "\n", "result = litellm.completion('claude-2', messages, stream=True)\n", "for part in result:\n", " print(part.choices[0].delta.content or \"\")\n", "\n" ] } ], "metadata": { "colab": { "provenance": [] }, "kernelspec": { "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python" } }, "nbformat": 4, "nbformat_minor": 0 }