Hosting_Llama2_Code_7B_On_Colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/UmarIgan/Machine-Learning/blob/master/Hosting_Llama2_Code_7B_On_Colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Llama 2 Code Streamlit app with Free GPU via Google Collab**\n",
        "- motivation: https://huggingface.co/blog/codellama\n",
        "\n",
        "---\n",
        "\n",
        "- First make sure you have gpu on on you colab\n",
        "- Install requirements for running  model and streamlit"
      ],
      "metadata": {
        "id": "bznyRLfLg750"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Install requirements for running  model and streamlit\n"
      ],
      "metadata": {
        "id": "kU0GYiR4hWKJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "u24fGvOmT2Pi",
        "outputId": "0241137d-d2e7-4906-b991-b821448abcc5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting streamlit\n",
            "  Downloading streamlit-1.26.0-py2.py3-none-any.whl (8.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.1/8.1 MB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-multipart\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.32.0-py3-none-any.whl (7.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.5/7.5 MB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate\n",
            "  Downloading accelerate-0.22.0-py3-none-any.whl (251 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.2/251.2 kB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting bitsandbytes\n",
            "  Downloading bitsandbytes-0.41.1-py3-none-any.whl (92.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.6/92.6 MB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pyngrok==4.1.1\n",
            "  Downloading pyngrok-4.1.1.tar.gz (18 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (0.18.3)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pyngrok==4.1.1) (6.0.1)\n",
            "Requirement already satisfied: altair<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (4.2.2)\n",
            "Requirement already satisfied: blinker<2,>=1.0.0 in /usr/lib/python3/dist-packages (from streamlit) (1.4)\n",
            "Requirement already satisfied: cachetools<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (5.3.1)\n",
            "Requirement already satisfied: click<9,>=7.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (8.1.7)\n",
            "Requirement already satisfied: importlib-metadata<7,>=1.4 in /usr/local/lib/python3.10/dist-packages (from streamlit) (6.8.0)\n",
            "Requirement already satisfied: numpy<2,>=1.19.3 in /usr/local/lib/python3.10/dist-packages (from streamlit) (1.23.5)\n",
            "Requirement already satisfied: packaging<24,>=16.8 in /usr/local/lib/python3.10/dist-packages (from streamlit) (23.1)\n",
            "Requirement already satisfied: pandas<3,>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (1.5.3)\n",
            "Requirement already satisfied: pillow<10,>=7.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (9.4.0)\n",
            "Requirement already satisfied: protobuf<5,>=3.20 in /usr/local/lib/python3.10/dist-packages (from streamlit) (3.20.3)\n",
            "Requirement already satisfied: pyarrow>=6.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (9.0.0)\n",
            "Collecting pympler<2,>=0.9 (from streamlit)\n",
            "  Downloading Pympler-1.0.1-py3-none-any.whl (164 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m164.8/164.8 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil<3,>=2.7.3 in /usr/local/lib/python3.10/dist-packages (from streamlit) (2.8.2)\n",
            "Requirement already satisfied: requests<3,>=2.18 in /usr/local/lib/python3.10/dist-packages (from streamlit) (2.31.0)\n",
            "Requirement already satisfied: rich<14,>=10.14.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (13.5.2)\n",
            "Requirement already satisfied: tenacity<9,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (8.2.3)\n",
            "Requirement already satisfied: toml<2,>=0.10.1 in /usr/local/lib/python3.10/dist-packages (from streamlit) (0.10.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.1.0 in /usr/local/lib/python3.10/dist-packages (from streamlit) (4.7.1)\n",
            "Collecting tzlocal<5,>=1.1 (from streamlit)\n",
            "  Downloading tzlocal-4.3.1-py3-none-any.whl (20 kB)\n",
            "Collecting validators<1,>=0.2 (from streamlit)\n",
            "  Downloading validators-0.21.2-py3-none-any.whl (25 kB)\n",
            "Collecting gitpython!=3.1.19,<4,>=3.0.7 (from streamlit)\n",
            "  Downloading GitPython-3.1.32-py3-none-any.whl (188 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.5/188.5 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pydeck<1,>=0.8 (from streamlit)\n",
            "  Downloading pydeck-0.8.0-py2.py3-none-any.whl (4.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m63.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tornado<7,>=6.0.3 in /usr/local/lib/python3.10/dist-packages (from streamlit) (6.3.2)\n",
            "Collecting watchdog>=2.1.5 (from streamlit)\n",
            "  Downloading watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.15.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m29.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m66.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m70.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit) (0.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit) (3.1.2)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit) (4.19.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit) (0.12.0)\n",
            "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.19,<4,>=3.0.7->streamlit)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7,>=1.4->streamlit) (3.16.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3,>=1.3.0->streamlit) (2023.3)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3,>=2.7.3->streamlit) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.18->streamlit) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.18->streamlit) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.18->streamlit) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.18->streamlit) (2023.7.22)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.14.0->streamlit) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.14.0->streamlit) (2.16.1)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
            "Collecting pytz-deprecation-shim (from tzlocal<5,>=1.1->streamlit)\n",
            "  Downloading pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.19,<4,>=3.0.7->streamlit)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->altair<6,>=4.0->streamlit) (2.1.3)\n",
            "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (23.1.0)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit) (0.9.2)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14,>=10.14.0->streamlit) (0.1.2)\n",
            "Collecting tzdata (from pytz-deprecation-shim->tzlocal<5,>=1.1->streamlit)\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
            "Building wheels for collected packages: pyngrok\n",
            "  Building wheel for pyngrok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyngrok: filename=pyngrok-4.1.1-py3-none-any.whl size=15963 sha256=f7906f9656842333f80fe6e13a0515a10552b4a3df085424a3d897846931a184\n",
            "  Stored in directory: /root/.cache/pip/wheels/4c/7c/4c/632fba2ea8e88d8890102eb07bc922e1ca8fa14db5902c91a8\n",
            "Successfully built pyngrok\n",
            "Installing collected packages: tokenizers, safetensors, bitsandbytes, watchdog, validators, tzdata, smmap, python-multipart, pyngrok, pympler, pytz-deprecation-shim, pydeck, huggingface-hub, gitdb, tzlocal, transformers, gitpython, streamlit, accelerate\n",
            "  Attempting uninstall: tzlocal\n",
            "    Found existing installation: tzlocal 5.0.1\n",
            "    Uninstalling tzlocal-5.0.1:\n",
            "      Successfully uninstalled tzlocal-5.0.1\n",
            "Successfully installed accelerate-0.22.0 bitsandbytes-0.41.1 gitdb-4.0.10 gitpython-3.1.32 huggingface-hub-0.16.4 pydeck-0.8.0 pympler-1.0.1 pyngrok-4.1.1 python-multipart-0.0.6 pytz-deprecation-shim-0.1.0.post0 safetensors-0.3.3 smmap-5.0.0 streamlit-1.26.0 tokenizers-0.13.3 transformers-4.32.0 tzdata-2023.3 tzlocal-4.3.1 validators-0.21.2 watchdog-3.0.0\n"
          ]
        }
      ],
      "source": [
        "!pip install streamlit python-multipart transformers accelerate bitsandbytes pyngrok==4.1.1"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "writing streamlit app file, %%writefile app.py command will help you with creating app file"
      ],
      "metadata": {
        "id": "2q92_8o_Bi50"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile app.py\n",
        "import streamlit as st\n",
        "from threading import Thread\n",
        "from typing import Iterator\n",
        "import torch\n",
        "import accelerate\n",
        "from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer\n",
        "\n",
        "\n",
        "model_id = 'codellama/CodeLlama-7b-Instruct-hf'\n",
        "cache_dir = 'model'\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    config = AutoConfig.from_pretrained(model_id,cache_dir=cache_dir)\n",
        "    config.pretraining_tp = 1\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_id,\n",
        "        config=config,\n",
        "        torch_dtype=torch.float16,\n",
        "        load_in_4bit=True,\n",
        "        device_map='auto',\n",
        "        use_safetensors=False,\n",
        "    )\n",
        "else:\n",
        "    model = None\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id,cache_dir=cache_dir)\n",
        "\n",
        "\n",
        "DEFAULT_SYSTEM_PROMPT = \"\"\"\\\n",
        "You are a helpful, respectful and honest assistant with a deep knowledge of code and software design.\n",
        "Always answer as helpfully as possible, while being safe. Your answers should not include any harmful,\n",
        "unethical, racist, sexist, toxic, dangerous, or illegal content.\n",
        "Please ensure that your responses are socially unbiased and positive in nature.\n",
        "\\n\\nIf a question does not make any sense, or is not factually coherent,\n",
        "explain why instead of answering something not correct.\n",
        "If you don't know the answer to a question, please don't share false information.\\\n",
        "\"\"\"\n",
        "MAX_MAX_NEW_TOKENS = 4096\n",
        "DEFAULT_MAX_NEW_TOKENS = 1024\n",
        "MAX_INPUT_TOKEN_LENGTH = 4000\n",
        "\n",
        "\n",
        "\n",
        "def get_prompt(message: str, chat_history: list[tuple[str, str]],\n",
        "               system_prompt: str) -> str:\n",
        "    texts = [f'<s>[INST] <<SYS>>\\n{system_prompt}\\n<</SYS>>\\n\\n']\n",
        "    # The first user input is _not_ stripped\n",
        "    do_strip = False\n",
        "    for user_input, response in chat_history:\n",
        "        user_input = user_input.strip() if do_strip else user_input\n",
        "        do_strip = True\n",
        "        texts.append(f'{user_input} [/INST] {response.strip()} </s><s>[INST] ')\n",
        "    message = message.strip() if do_strip else message\n",
        "    texts.append(f'{message} [/INST]')\n",
        "    return ''.join(texts)\n",
        "\n",
        "def get_input_token_length(message: str, chat_history: list[tuple[str, str]], system_prompt: str) -> int:\n",
        "    prompt = get_prompt(message, chat_history, system_prompt)\n",
        "    input_ids = tokenizer([prompt], return_tensors='np', add_special_tokens=False)['input_ids']\n",
        "    return input_ids.shape[-1]\n",
        "\n",
        "\n",
        "def run(message: str,\n",
        "        chat_history: list[tuple[str, str]],\n",
        "        system_prompt: str,\n",
        "        max_new_tokens: int = 1024,\n",
        "        temperature: float = 0.1,\n",
        "        top_p: float = 0.9,\n",
        "        top_k: int = 50) -> Iterator[str]:\n",
        "    prompt = get_prompt(message, chat_history, system_prompt)\n",
        "    inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')\n",
        "\n",
        "    streamer = TextIteratorStreamer(tokenizer,\n",
        "                                    timeout=10.,\n",
        "                                    skip_prompt=True,\n",
        "                                    skip_special_tokens=True)\n",
        "    generate_kwargs = dict(\n",
        "        inputs,\n",
        "        streamer=streamer,\n",
        "        do_sample=True,\n",
        "        max_new_tokens = 1024,\n",
        "        temperature = 0.1,\n",
        "        top_p = 0.9,\n",
        "        top_k = 50,\n",
        "        num_beams=1)\n",
        "    t = Thread(target=model.generate, kwargs=generate_kwargs)\n",
        "    t.start()\n",
        "\n",
        "    outputs = []\n",
        "    for text in streamer:\n",
        "        outputs.append(text)\n",
        "        yield ''.join(outputs)\n",
        "\n",
        "def main():\n",
        "    st.title(\"Code Llama 7B Chat on Colab GPU by Umar IGAN\")\n",
        "    st.markdown(\"\"\"\n",
        "    This Space demonstrates model [CodeLlama-7b-Instruct](https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf) by Meta, a Code Llama model with 7B parameters fine-tuned for chat instructions and specialized on code tasks.\n",
        "    I build this on colab in a way to use your colab gpu on this interface.\n",
        "    \"\"\")\n",
        "\n",
        "    message = st.text_area(\"Type a message...\")\n",
        "    system_prompt = st.text_area(\"System prompt\", value=DEFAULT_SYSTEM_PROMPT)\n",
        "\n",
        "    if st.button(\"Submit\"):\n",
        "        history = []\n",
        "        input_token_length = get_input_token_length(message, history, system_prompt)\n",
        "        if input_token_length > MAX_INPUT_TOKEN_LENGTH:\n",
        "            st.error(f'The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Clear your chat history and try again.')\n",
        "            return\n",
        "\n",
        "        output_elem = st.empty()  # Create an empty element for dynamic text update\n",
        "\n",
        "        def update_output(response):\n",
        "            history.append((message, response))\n",
        "            output_elem.text(response)  # Update the element's content with the response\n",
        "\n",
        "        generator = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)\n",
        "        for response in generator:\n",
        "            #st.write(response)  # Display response in the sidebar as well\n",
        "            update_output(response)  # Update the dynamic text element with the response\n",
        "\n",
        "\n",
        "    if st.button(\"Clear\"):\n",
        "        st.text_area(\"Type a message...\", value=\"\")\n",
        "\n",
        "    if st.button(\"Retry\"):\n",
        "        pass  # Implement the retry logic here\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    main()\n"
      ],
      "metadata": {
        "id": "Z6F078OHVssf",
        "outputId": "ce485716-3744-41f6-ce85-e23791d904ff",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Writing app.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "we added log file to track download process of model files."
      ],
      "metadata": {
        "id": "Z__SaEjQsqjy"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "You can get a public url running codes below.\n",
        "\n"
      ],
      "metadata": {
        "id": "Zgb1YX_qCDSy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!npm install localtunnel"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ABDAO-XLmTn",
        "outputId": "8116b890-2630-46a2-d173-4f0c0acb1cf6"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K\u001b[?25h\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m \u001b[0m\u001b[35msaveError\u001b[0m ENOENT: no such file or directory, open '/content/package.json'\n",
            "\u001b[K\u001b[?25h\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[34;40mnotice\u001b[0m\u001b[35m\u001b[0m created a lockfile as package-lock.json. You should commit this file.\n",
            "\u001b[0m\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m \u001b[0m\u001b[35menoent\u001b[0m ENOENT: no such file or directory, open '/content/package.json'\n",
            "\u001b[0m\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m\u001b[35m\u001b[0m content No description\n",
            "\u001b[0m\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m\u001b[35m\u001b[0m content No repository field.\n",
            "\u001b[0m\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m\u001b[35m\u001b[0m content No README data\n",
            "\u001b[0m\u001b[37;40mnpm\u001b[0m \u001b[0m\u001b[30;43mWARN\u001b[0m\u001b[35m\u001b[0m content No license field.\n",
            "\u001b[0m\n",
            "+ localtunnel@2.0.2\n",
            "added 22 packages from 22 contributors and audited 22 packages in 2.68s\n",
            "\n",
            "3 packages are looking for funding\n",
            "  run `npm fund` for details\n",
            "\n",
            "found \u001b[92m0\u001b[0m vulnerabilities\n",
            "\n",
            "\u001b[K\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!streamlit run app.py &>/content/logs.txt &"
      ],
      "metadata": {
        "id": "xsPPstiDGB95"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Get your ip address"
      ],
      "metadata": {
        "id": "wEiHP_LARHT_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget -q -O - ipv4.icanhazip.com"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPhcFFUvOq6L",
        "outputId": "193dec94-f7bd-4a44-eb89-b1c57594e20d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "34.30.29.231\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!npx localtunnel --port 8501"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yBkrFvzmOxpo",
        "outputId": "093be02f-4422-4513-e09c-f3ba3fd5040f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K\u001b[?25hnpx: installed 22 in 2.008s\n",
            "your url is: https://ready-emus-attack.loca.lt\n"
          ]
        }
      ]
    }
  ]
}