assets/notebooks/OpenAI_Bioinformatics_Tool_Embeddings.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "__kZPTe2GU1h",
        "outputId": "2a25ca1d-5b15-4daf-b1dc-aec9a71b0855"
      },
      "outputs": [],
      "source": [
        "#@title Setup & Installation { display-mode: \"form\" }\n",
        "\n",
        "!sudo apt install tesseract-ocr -q\n",
        "!sudo apt-get install poppler-utils -q\n",
        "!pip install -qq pytesseract pdf2image zero-shot-re neo4j boto3 datasets\n",
        "!pip -qq install boto3\n",
        "!pip -qq install pyyaml==5.4.1\n",
        "!pip -qq install astrodendro\n",
        "!pip -qq install openai\n",
        "!pip -qq install igraph\n",
        "!pip -qq install trimap\n",
        "!pip -qq install transformers\n",
        "import boto3\n",
        "from botocore.exceptions import NoCredentialsError"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bjOragzYi8Be"
      },
      "outputs": [],
      "source": [
        "%config InlineBackend.figure_format = \"retina\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FcujmpYa5ewf"
      },
      "source": [
        "# Downloading nodes from AWS\n",
        "\n",
        "We're downloading these from an AWS bucket, so we can feed the JSON strings into the Code Embedding endpoint."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCMrIuWp29v4"
      },
      "source": [
        "### Download from S3 bucket"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W3JfXg54-qSF",
        "outputId": "aa8479d4-ffc0-45d9-ca65-319b04312f0c"
      },
      "outputs": [],
      "source": [
        "import boto3\n",
        "from botocore.exceptions import NoCredentialsError\n",
        "\n",
        "from getpass import getpass\n",
        "\n",
        "ACCESS_KEY = getpass(prompt='AWS ACCESS KEY: ')\n",
        "SECRET_KEY = getpass(prompt='AWS SECRET KEY: ')\n",
        "\n",
        "\n",
        "def download_from_aws(local_file: str, bucket: str, s3_file: str):\n",
        "    session = boto3.Session(\n",
        "        aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY\n",
        "    )\n",
        "    s3 = session.resource(\"s3\")\n",
        "    try:\n",
        "        s3.Bucket(bucket).download_file(s3_file, local_file)\n",
        "        print(\n",
        "            f\"Download of {s3_file} from bucket {bucket} Successful.\\n\",\n",
        "            f\"Saved as {local_file}\"\n",
        "        )\n",
        "        return True\n",
        "    except FileNotFoundError:\n",
        "        print(\"The file was not found\")\n",
        "        return False\n",
        "    except NoCredentialsError:\n",
        "        print(\"Credentials not available\")\n",
        "        return False\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uyC5tltXj4Pt",
        "outputId": "6c18fe66-443b-440f-aced-ef3d24931a3d"
      },
      "outputs": [],
      "source": [
        "download = download_from_aws(\n",
        "    \"node_records_Biotool.json\",\n",
        "    \"knowledge-graph-backup\",\n",
        "    \"node_records_Biotool.json\",\n",
        ")\n",
        "download = download_from_aws(\n",
        "    \"node_records_Toolset.json\",\n",
        "    \"knowledge-graph-backup\",\n",
        "    \"node_records_Toolset.json\",\n",
        ")\n",
        "download = download_from_aws(\n",
        "    \"relationship_records_FEEDS_INTO.json\",\n",
        "    \"knowledge-graph-backup\",\n",
        "    \"relationship_records_FEEDS_INTO.json\",\n",
        ")\n",
        "download = download_from_aws(\n",
        "    \"biotools_node_openai_embeddings.csv\",\n",
        "    \"knowledge-graph-backup\",\n",
        "    \"biotools_node_openai_embeddings.csv\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Yzgm3Lp-qMn",
        "outputId": "640ba592-250a-4f0d-c3c2-803090d85fac"
      },
      "outputs": [],
      "source": [
        "import codecs\n",
        "import json\n",
        "import time\n",
        "\n",
        "def load_json_dict_from_file(file_name):\n",
        "    \"\"\"\n",
        "    Loads entity list from json file\n",
        "    \"\"\"\n",
        "    return json.load(codecs.open(file_name, 'r', 'utf-8-sig'), strict=False)\n",
        "\n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "\n",
        "node_records_Biotool = load_json_dict_from_file(\"/content/node_records_Biotool.json\")\n",
        "node_records_Toolset = load_json_dict_from_file(\"/content/node_records_Toolset.json\")\n",
        "relationship_records_FEEDS_INTO = load_json_dict_from_file(\"/content/relationship_records_FEEDS_INTO.json\")\n",
        "\n",
        "end_time = time.time()\n",
        "print(\"Time taken: {:.4g} seconds\".format(end_time - start_time))\n",
        "#named_enities\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SidccKlT5h0N"
      },
      "source": [
        "# Importing nodes into Neo4j\n",
        "\n",
        "This guide here details importing data. This is one of the hyperlinks that shows up when starting a blank sandbox.\n",
        "\n",
        "https://neo4j.com/developer/guide-importing-data-and-etl/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wNpbC9KBpqVL",
        "outputId": "917efe1f-7c26-4d1d-8ad3-f9e3ba5f1b1b"
      },
      "outputs": [],
      "source": [
        "# Creating dictionaries corresponding to each node type\n",
        "\n",
        "# Each of the nodes in the backup are indexed.\n",
        "# All the relationships have details on which indexes they go between\n",
        "# This makes sure all the nodes can be easily uploaded\n",
        "\n",
        "exclude_keys = [\"identity\", \"labels\",\"created_by\", \"created_time\"]\n",
        "\n",
        "node_records_Biotool_dict = {}\n",
        "node_records_Toolset_dict = {}\n",
        "\n",
        "\n",
        "for node_record in node_records_Biotool:\n",
        "    identity_key = node_record[\"n\"][\"identity\"]\n",
        "    new_node_record = {\n",
        "        k: node_record[\"n\"][\"properties\"][k]\n",
        "        for k in set(list(node_record[\"n\"][\"properties\"].keys()))\n",
        "        - set(exclude_keys)\n",
        "    }\n",
        "    node_records_Biotool_dict.update({identity_key: new_node_record})\n",
        "\n",
        "for node_record in node_records_Toolset:\n",
        "    identity_key = node_record[\"a\"][\"identity\"]\n",
        "    new_node_record = {\n",
        "        k: node_record[\"a\"][\"properties\"][k]\n",
        "        for k in set(list(node_record[\"a\"][\"properties\"].keys()))\n",
        "        - set(exclude_keys)\n",
        "    }\n",
        "    node_records_Toolset_dict.update({identity_key: new_node_record})\n",
        "\n",
        "print(\"Number of Biotool: \", len(node_records_Biotool_dict))\n",
        "print(\"Number of Toolset: \", len(node_records_Toolset_dict))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0SDYRH_eiCwc",
        "outputId": "6c45bcc2-179e-4f9a-c23f-be0fbdeacc35"
      },
      "outputs": [],
      "source": [
        "exclude_keys = [\"source\", \"target\", \"caption\"]\n",
        "\n",
        "# FEEDS_INTO\n",
        "relationship_records_FEEDS_INTO_list = []\n",
        "\n",
        "for rel_record in relationship_records_FEEDS_INTO[0][\"edgesFEEDS_INTO\"]:\n",
        "    source_id = rel_record[\"source\"]\n",
        "    target_id = rel_record[\"target\"]\n",
        "    r_type = rel_record[\"r_type\"]\n",
        "    weight = rel_record[\"weight\"] + 1\n",
        "\n",
        "    #new_rel_record = {\n",
        "    #    k: rel_record[k]\n",
        "    #    for k in set(list(rel_record.keys())) - set(exclude_keys)\n",
        "    #}\n",
        "    relationship_records_FEEDS_INTO_list.append(\n",
        "        #{\"source\": source_id, \"target\": target_id, \"data_category\":r_type,\"weight\":weight}\n",
        "        {\"source\": source_id, \"target\": target_id, \"weight\":weight}\n",
        "    )\n",
        "\n",
        "print(\"Number of FEEDS_INTO: \", len(relationship_records_FEEDS_INTO_list))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d3EZ2nxxW2pP"
      },
      "source": [
        "### All `BioTool` nodes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bZMvmX7kP0TI",
        "outputId": "5d6483d4-3c2c-4ae5-980d-fc20033d8de3"
      },
      "outputs": [],
      "source": [
        "node_records_Biotool_dict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zIGynwuMirGQ",
        "outputId": "11711a4f-ba71-4e46-fec4-a11b4072aa99"
      },
      "outputs": [],
      "source": [
        "node_records_Toolset_dict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RKg7lGypiu67",
        "outputId": "5892b364-1fb2-4c92-97fd-4b9942477687"
      },
      "outputs": [],
      "source": [
        "relationship_records_FEEDS_INTO_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "u4tQWAS3igaM",
        "outputId": "fa39f1c3-2417-4103-bcf4-0d771e648097"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from typing import Dict\n",
        "import json\n",
        "\n",
        "def dict_to_pandas_dataframe(dict_data: Dict[int, Dict]):\n",
        "    \"\"\"\n",
        "    Takes in a dictionary of lists and converts it to a pandas dataframe.\n",
        "    for the first column, this is all the list elements from the dictionary.\n",
        "    for the second column, this is the value's key from when it was part of the dictionary.\n",
        "    \"\"\"\n",
        "    df_list = []\n",
        "    for key, value in dict_data.items():\n",
        "        df_list.append([key, value['name'], value['operation'], value['language'], int(value['citationCounts']),json.dumps(value)])\n",
        "    return pd.DataFrame(df_list, columns=['protocol_id', 'protocol_name', 'protocol_operation', 'protocol_language', 'protocol_citationCounts', 'protocol_value'])\n",
        "\n",
        "df_old = dict_to_pandas_dataframe(node_records_Biotool_dict)\n",
        "df_old.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UTHP7wlCjQym",
        "outputId": "d1cbe3bf-7bd6-4e21-9545-77185fa40049"
      },
      "outputs": [],
      "source": [
        "len(df_old.protocol_id.unique())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5YHXD8JpigXG",
        "outputId": "79801daf-f472-4f42-8aab-6f977205394c"
      },
      "outputs": [],
      "source": [
        "len(df_old)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xh4oSZKJjU4C"
      },
      "source": [
        "### Getting the embeddings\n",
        "\n",
        "https://beta.openai.com/docs/guides/embeddings/use-cases?lang=python\n",
        "\n",
        "https://github.com/openai/openai-python/tree/0b07e1e9457d3a6252431036b725b3e891f95a40/examples/embeddings\n",
        "\n",
        "https://stackabuse.com/hierarchical-clustering-with-python-and-scikit-learn/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "LiUE1RA4igUd",
        "outputId": "ba55efbb-fd72-4e13-ab9f-6a35d30e109b"
      },
      "outputs": [],
      "source": [
        "from transformers import GPT2TokenizerFast\n",
        "tokenizer = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n",
        "\n",
        "# remove reviews that are too long\n",
        "df['n_tokens'] = df.protocol_value.apply(lambda x: len(tokenizer.encode(x)))\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K6hVeO3pTaTE",
        "outputId": "80bdc2e7-4339-468b-aae7-c9833295b3a9"
      },
      "outputs": [],
      "source": [
        "total_tokens = df['n_tokens'].sum()\n",
        "\n",
        "print(\"Price of tokenization using Ada Embeddings (Fastest): $\", (total_tokens/1000 ) * 0.0040)\n",
        "print(\"Price of tokenization using Babbage Embeddings: $\", (total_tokens/1000 ) * 0.0050)\n",
        "print(\"Price of tokenization using Curie Embeddings: $\", (total_tokens/1000 ) * 0.0200)\n",
        "print(\"Price of tokenization using DaVinci Embeddings (Most powerful): $\", (total_tokens/1000 ) * 0.2000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DK5JHVaTkFXf",
        "outputId": "01f8e510-3f8a-42a5-d570-52656e7e8752"
      },
      "outputs": [],
      "source": [
        "total_tokens"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rs6bv9RvigSI",
        "outputId": "cf971111-91ec-40fc-888c-516600b12c88"
      },
      "outputs": [],
      "source": [
        "from numpy.core.numeric import True_\n",
        "import openai\n",
        "from getpass import getpass\n",
        "\n",
        "CREATE_NEW_EMBEDDINGS = False #@param\n",
        "EMBEDDING_ENGINE = 'code-search-ada-code-001' #@param [\"code-search-ada-code-001\", \"code-search-ada-text-001\", \"code-search-babbage-code-001\", \"code-search-babbage-text-001\", \"code-search-babbage-code-001\", \"text-similarity-babbage-001\", \"text-search-babbage-doc-001\"] {allow-input: false}\n",
        "\n",
        "if CREATE_NEW_EMBEDDINGS==True:\n",
        "    openai.api_key = getpass(prompt='OpenAI Access Key: ')\n",
        "\n",
        "    def get_embedding(input_text, engine=None):\n",
        "        if engine is None:\n",
        "            raise ValueError(\"Need to choose engine\")\n",
        "\n",
        "        response = openai.Embedding.create(\n",
        "            input=input_text,\n",
        "            engine=engine\n",
        "        )\n",
        "        embeddings = response['data'][0]['embedding']\n",
        "        return embeddings\n",
        "    \n",
        "    df[EMBEDDING_ENGINE] = df.protocol_value.apply(lambda x: get_embedding(x, engine=EMBEDDING_ENGINE))\n",
        "    df.head()\n",
        "else:\n",
        "    print(\"Skipping the creation of new embeddings\")\n",
        "    #df = pd.read_csv('/content/biotools_node_openai_embeddings.csv')\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "neermQzXgHRI"
      },
      "source": [
        "```python\n",
        "# mount it\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# copy it there\n",
        "!cp biotools_node_openai_embeddings.csv /content/drive/MyDrive\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9EDrOHj7j2Wo"
      },
      "source": [
        "## Dimensionality Reduction\n",
        "\n",
        "Use tri-map here\n",
        "\n",
        "https://github.com/openai/openai-python/blob/main/examples/embeddings/Clustering.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F5K-_aFOigJR",
        "outputId": "fff797ae-3b11-41f5-995b-5589c748907a"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.manifold import TSNE\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib\n",
        "\n",
        "\n",
        "df = pd.read_csv('/content/biotools_node_openai_embeddings.csv')\n",
        "df['code-search-ada-code-001'] = df['code-search-ada-code-001'].apply(eval).apply(np.array)\n",
        "df['code-search-ada-text-001'] = df['code-search-ada-text-001'].apply(eval).apply(np.array)\n",
        "matrix_code = np.vstack(df['code-search-ada-code-001'].values)\n",
        "matrix_text = np.vstack(df['code-search-ada-text-001'].values)\n",
        "matrix_code.shape, matrix_text.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "guTLYIeaigGp"
      },
      "outputs": [],
      "source": [
        "from sklearn.cluster import KMeans\n",
        "\n",
        "n_clusters = 30\n",
        "\n",
        "kmeans = KMeans(n_clusters = n_clusters,init='k-means++',random_state=42)\n",
        "kmeans.fit(matrix_text)\n",
        "labels = kmeans.labels_\n",
        "df['Cluster'] = labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PGz8ZB0zigEB",
        "outputId": "86fc5859-ce99-4de6-dbc9-1d75b923ed69"
      },
      "outputs": [],
      "source": [
        "unique_values = df['Cluster'].unique()\n",
        "unique_values\n",
        "\n",
        "\n",
        "np.where(unique_values == 'Get Data')[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VHojCBp1uj0-",
        "outputId": "d4762754-5997-434c-c533-dc77b61d2289"
      },
      "outputs": [],
      "source": [
        "relationship_records_FEEDS_INTO_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LfwenUDQNrBk"
      },
      "outputs": [],
      "source": [
        "import igraph as ig\n",
        "\n",
        "Edges = [\n",
        "    (\n",
        "        relationship_records_FEEDS_INTO_list[k][\"source\"],\n",
        "        relationship_records_FEEDS_INTO_list[k][\"target\"],\n",
        "        relationship_records_FEEDS_INTO_list[k][\"weight\"],\n",
        "    )\n",
        "    for k in range(len(relationship_records_FEEDS_INTO_list))\n",
        "]\n",
        "\n",
        "G = ig.Graph.TupleList(Edges, directed=True, weights=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vj77wOn55ntR",
        "outputId": "b217a375-cbc7-4a00-e796-cfce23e47be3"
      },
      "outputs": [],
      "source": [
        "Edges"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qCKn01BrJM5t"
      },
      "source": [
        "### [TriMAP Documentation](https://github.com/eamid/trimap)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-RfvifXSh1Ae"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "\n",
        "colors_list = [\n",
        "    \"#201923\",\n",
        "    \"#888888\",\n",
        "    \"#fcff5d\",\n",
        "    \"#7dfc00\",\n",
        "    \"#0ec434\",\n",
        "    \"#228c68\",\n",
        "    \"#8ad8e8\",\n",
        "    \"#235b54\",\n",
        "    \"#29bdab\",\n",
        "    \"#3998f5\",\n",
        "    \"#37294f\",\n",
        "    \"#277da7\",\n",
        "    \"#3750db\",\n",
        "    \"#f22020\",\n",
        "    \"#991919\",\n",
        "    \"#ffcba5\",\n",
        "    \"#e68f66\",\n",
        "    \"#c56133\",\n",
        "    \"#96341c\",\n",
        "    \"#632819\",\n",
        "    \"#ffc413\",\n",
        "    \"#f47a22\",\n",
        "    \"#2f2aa0\",\n",
        "    \"#b732cc\",\n",
        "    \"#772b9d\",\n",
        "    \"#f07cab\",\n",
        "    \"#d30b94\",\n",
        "    \"#c3a5b4\",\n",
        "    \"#946aa2\",\n",
        "    \"#5d4c86\",\n",
        "]\n",
        "\n",
        "\n",
        "languages_dict = {\n",
        "    \"D\": \"#ba595e\", \"PHP\": \"#4F5D95\", \"Racket\": \"#3c5caa\", \"Smalltalk\": \"#596706\",\n",
        "    \"Groovy\": \"#4298b8\", \"Scala\": \"#c22d40\", \"LabVIEW\": \"#fede06\", \"R\": \"#198CE7\",\n",
        "    \"Pascal\": \"#E3F171\", \"Forth\": \"#341708\", \"Perl\": \"#0298c3\", \"Ruby\": '#701516\"',\n",
        "    \"JSP\": \"#2A6277\", \"C++\": \"#f34b7d\", \"JavaScript\": \"#f1e05a\", \"Mathematica\": \"#dd1100\",\n",
        "    \"Verilog\": \"#b2b7f8\", \"VHDL\": \"#adb2cb\", \"Lua\": \"#000080\", \"C#\": \"#178600\",\n",
        "    \"AWK\": \"#c30e9b\", \"Lisp\": \"#3fb68b\", \"SQL\": \"#e38c00\", \"Java\": \"#b07219\",\n",
        "    \"OCaml\": \"#3be133\", \"Haskell\": \"#5e5086\", \"Maple\": \"#c22d40\", \"MATLAB\": \"#e16737\",\n",
        "    \"Prolog\": \"#74283c\", \"Other\": \"#cccccc\", \"Delphi\": \"#E3F171\", \"Elm\": \"#60B5CC\",\n",
        "    \"Julia\": \"#a270ba\", \"Scheme\": \"#1e4aec\", \"Ada\": \"#02f88c\", \"PyMOL\": \"807F01\",\n",
        "    \"ActionScript\": \"#882B0F\", \"Fortran\": \"#4d41b1\", \"Visual Basic\": \"#945db7\",\n",
        "    \"Bash\": \"#89e051\", \"Python\": \"#3572A5\", \"Shell\": \"#89e051\", \"SAS\": \"#B34936\",\n",
        "    \"CWL\": \"#B5314C\", \"C\": \"#555555\", \"None\": \"#bbbbbb\",\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kHOH5iCBhg2I",
        "outputId": "ce72d772-6d3a-40f9-d9a2-b6f79e9aee46"
      },
      "outputs": [],
      "source": [
        "hex_to_rgb('#c22d40')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DCw4KTFVif-j"
      },
      "outputs": [],
      "source": [
        "\n",
        "def hex_to_rgb(hex_color: str):\n",
        "    h = hex_color.lstrip(\"#\")\n",
        "    try:\n",
        "        r = int(h[0 : 2], 16)\n",
        "        g = int(h[2 : 4], 16)\n",
        "        b = int(h[4 : 6], 16)\n",
        "    except:\n",
        "        raise(ValueError(f\"Error with input {h}\"))\n",
        "\n",
        "    return tuple((r, g, b))\n",
        "\n",
        "\n",
        "def average_rgb_colors(rgb_colors: list):\n",
        "    r = 0\n",
        "    g = 0\n",
        "    b = 0\n",
        "    for color in rgb_colors:\n",
        "        r += color[0]\n",
        "        g += color[1]\n",
        "        b += color[2]\n",
        "    r = int(r / len(rgb_colors))\n",
        "    g = int(g / len(rgb_colors))\n",
        "    b = int(b / len(rgb_colors))\n",
        "    return (r, g, b)\n",
        "\n",
        "\n",
        "def rgb_to_hex(rgb):\n",
        "    return \"#{:02x}{:02x}{:02x}\".format(rgb[0], rgb[1], rgb[2])\n",
        "\n",
        "\n",
        "def average_hex_colors(hex_colors: list):\n",
        "    # convert hex colors to rgb\n",
        "    rgb_colors = [hex_to_rgb(hex_color) for hex_color in hex_colors]\n",
        "    # average the rgb colors\n",
        "    avg_rgb_color = average_rgb_colors(rgb_colors)\n",
        "    # convert the average rgb color back to hex\n",
        "    avg_hex_color = rgb_to_hex(avg_rgb_color)\n",
        "    return avg_hex_color\n",
        "\n",
        "\n",
        "def correct_hex_format(hex_color_string: str):\n",
        "    if not hex_color_string.startswith('#'):\n",
        "        hex_color_string = '#' + hex_color_string\n",
        "    if hex_color_string[-1] == '\"':\n",
        "        hex_color_string = hex_color_string[:-1]\n",
        "    return hex_color_string\n",
        "\n",
        "\n",
        "def add_language_colors_to_dataframe(\n",
        "    df: pd.DataFrame, language_color_dict: Dict[str, str]\n",
        "):\n",
        "    df[\"language_colors\"] = df[\"protocol_language\"].apply(\n",
        "        lambda x: language_color_dict[x]\n",
        "        if x in language_color_dict\n",
        "        else average_hex_colors([language_color_dict[lang] for lang in x.split(\",\")])\n",
        "    )\n",
        "    df[\"language_colors\"] = df[\"language_colors\"].apply(\n",
        "        lambda x: correct_hex_format(x)\n",
        "    )\n",
        "    return df\n",
        "\n",
        "def add_labeltext_to_dataframe(\n",
        "    df: pd.DataFrame,\n",
        "):\n",
        "    df[\"hovertext\"] = \"Name: \" + df[\"protocol_name\"] + \"<br>Operation Type: \" + df[\"protocol_operation\"] + \"<br>Language(s): \" + df[\"protocol_language\"]\n",
        "    return df\n",
        "\n",
        "\n",
        "def add_colors_to_dataframe(df: pd.DataFrame, col_name: str):\n",
        "    \"\"\"\n",
        "    For a given column, creates a color palette based on the unique values in the column.\n",
        "    adds a new column to the dataframe with the color palette.\n",
        "    \"\"\"\n",
        "    random.seed(42)\n",
        "    unique_values = df[col_name].unique()\n",
        "    colors = [random.choice(colors_list) for i in range(len(unique_values))]\n",
        "    df[col_name + \"_color\"] = df[col_name].map(\n",
        "        lambda x: colors_list[np.where(unique_values == x)[0][0]]\n",
        "    )\n",
        "    return df\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XkJpPeR_hfUn"
      },
      "outputs": [],
      "source": [
        "df = add_language_colors_to_dataframe(df, languages_dict)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1u-BmxwpqGa8"
      },
      "outputs": [],
      "source": [
        "df = add_labeltext_to_dataframe(df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 832
        },
        "id": "8GqIQDKsif77",
        "outputId": "a1b302d6-8f66-45ef-eb31-dd54dd9a7570"
      },
      "outputs": [],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PKQyesRlkFj4"
      },
      "source": [
        "https://github.com/eamid/examples/blob/master/TriMap.ipynb"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gw2NMjmC6fPA"
      },
      "source": [
        "### 3D Plotting"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4Y4EvWpYif07"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from scipy.cluster import hierarchy\n",
        "import matplotlib.pyplot as plt\n",
        "from bokeh.plotting import figure, output_file, show\n",
        "from bokeh.io import output_notebook\n",
        "\n",
        "# Call once to configure Bokeh to display plots inline in the notebook.\n",
        "output_notebook()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 699
        },
        "id": "TQeZ1ed16he9",
        "outputId": "f608115e-0071-417a-d2af-86df845111d8"
      },
      "outputs": [],
      "source": [
        "from sklearn.manifold import TSNE\n",
        "import matplotlib\n",
        "from mpl_toolkits import mplot3d\n",
        "import matplotlib.pyplot as plt\n",
        "import jax.random as random\n",
        "import trimap\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "#@markdown TriMAP parameters\n",
        "#@markdown n_dims: Number of dimensions of the embedding (default = 2)\n",
        "N_DIMS = 3 #@param\n",
        "#@markdown n_inliers: Number of nearest neighbors for forming the nearest neighbor triplets (default = 12).\n",
        "N_INLIERS = 12 #@param\n",
        "#@markdown n_outliers: Number of outliers for forming the nearest neighbor triplets (default = 4).\n",
        "N_OUTLIERS = 4 #@param\n",
        "#@markdown n_random: Number of random triplets per point (default = 3).\n",
        "N_RANDOM = 6 #@param\n",
        "#@markdown distance: Distance measure ('euclidean' (default), 'manhattan', 'angular' (or 'cosine'), 'hamming')\n",
        "DISTANCE = 'cosine' #@param\n",
        "#@markdown weight_temp: Temperature of the logarithm applied to the weights. Larger temperatures generate more compact embeddings. weight_temp=0. corresponds to no transformation (default=0.5).\n",
        "WEIGHT_TEMP = 0.5 #@param\n",
        "#@markdown weight_adj **(deprecated)**: The value of gamma for the log-transformation (default = 500.0).\n",
        "WEIGHT_ADJ = 500.0 #@param\n",
        "#@markdown lr: Learning rate (default = 0.1).\n",
        "LR = 0.1 #@param\n",
        "#@markdown n_iters: Number of iterations (default = 400).\n",
        "N_ITERS = 400 #@param\n",
        "#@markdown The other parameters include:\n",
        "#@markdown knn_tuple: Use the precomputed nearest-neighbors information in form of a tuple (knn_nbrs, knn_distances) (default = None)\n",
        "KNN_TUPLE = None #@param\n",
        "#@markdown use_dist_matrix: Use the precomputed pairwise distance matrix (default = False)\n",
        "USE_DIST_MATRIX = False #@param\n",
        "#@markdown apply_pca: Reduce the number of dimensions of the data to 100 if necessary before applying the nearest-neighbor search (default = True).\n",
        "APPLY_PCA = True #@param\n",
        "#@markdown opt_method: Optimization method {'sd' (steepest descent), 'momentum' (GD with momentum), 'dbd' (delta-bar-delta, default)}.\n",
        "OPT_METHOD = 'dbd' #@param\n",
        "#@markdown verbose: Print the progress report (default = False).\n",
        "VERBOSE = True #@param\n",
        "#@markdown return_seq: Store the intermediate results and return the results in a tensor (default = False).\n",
        "RETURN_SEQ = False #@param\n",
        "\n",
        "# TriMap\n",
        "key = random.PRNGKey(123)\n",
        "vis_dims3 = trimap.TRIMAP(\n",
        "    n_dims=N_DIMS,\n",
        "    n_inliers=N_INLIERS,\n",
        "    n_outliers=N_OUTLIERS,\n",
        "    n_random=N_RANDOM,\n",
        "    distance=DISTANCE,\n",
        "    weight_temp=WEIGHT_TEMP,\n",
        "    weight_adj=WEIGHT_ADJ,\n",
        "    lr=LR,\n",
        "    n_iters=N_ITERS,\n",
        "    knn_tuple=KNN_TUPLE,\n",
        "    use_dist_matrix=USE_DIST_MATRIX,\n",
        "    apply_pca=APPLY_PCA,\n",
        "    opt_method=OPT_METHOD,\n",
        "    verbose=VERBOSE,\n",
        "    return_seq=RETURN_SEQ,\n",
        ").fit_transform(matrix_code)\n",
        "\n",
        "# Define Data\n",
        " \n",
        "x = [x for x, y, z in vis_dims3]\n",
        "y = [y for x, y, z in vis_dims3]\n",
        "z = [z for x, y, z in vis_dims3]\n",
        "\n",
        "\n",
        "# Create Figure\n",
        "\n",
        "fig = plt.figure(figsize=(10, 7))\n",
        "ax = plt.axes(projection=\"3d\")\n",
        "\n",
        "# Create Plot\n",
        "\n",
        "# scatter3D(x, y, z, color=None)\n",
        "for category, color in enumerate(colors_list):\n",
        "    xs = np.array(x)[df.Cluster == category]\n",
        "    ys = np.array(y)[df.Cluster == category]\n",
        "    zs = np.array(z)[df.Cluster == category]\n",
        "    ax.scatter3D(xs, ys, zs, color=color, alpha=0.3)\n",
        "\n",
        "# Show plot\n",
        "plt.title(\"Clusters identified visualized in language 3D using TriMAP\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gD2umT536hl2"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from scipy.cluster import hierarchy\n",
        "import matplotlib.pyplot as plt\n",
        "from bokeh.plotting import figure, output_file, show\n",
        "from bokeh.io import output_notebook\n",
        "\n",
        "# Call once to configure Bokeh to display plots inline in the notebook.\n",
        "output_notebook()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lTGP8Q_vAdVS"
      },
      "outputs": [],
      "source": [
        "Xn = x  # x-coordinates of nodes\n",
        "Yn = y  # y-coordinates\n",
        "Zn = z  # z-coordinates\n",
        "Xe = []\n",
        "Ye = []\n",
        "Ze = []\n",
        "for e in Edges:\n",
        "    Xe += [vis_dims3[e[0]][0], vis_dims3[e[1]][0], None]  # x-coordinates of edge ends\n",
        "    Ye += [vis_dims3[e[0]][1], vis_dims3[e[1]][1], None]\n",
        "    Ze += [vis_dims3[e[0]][2], vis_dims3[e[1]][2], None]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k1-H2zGwonJG"
      },
      "source": [
        "https://plotly.com/python/v3/3d-network-graph/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "oUGetIe7AdN0",
        "outputId": "4f7d6f21-f733-4c21-c855-1a6b7ccb5dee"
      },
      "outputs": [],
      "source": [
        "import plotly as py\n",
        "import plotly.graph_objects as go\n",
        "\n",
        "#rs = np.random.RandomState()\n",
        "#rs.seed(0)\n",
        "#\n",
        "#fig = go.Figure(\n",
        "#    data=go.Scatter3d(\n",
        "#        x=x,\n",
        "#        y=y,\n",
        "#        z=z,\n",
        "#        marker=dict(\n",
        "#            size=2,\n",
        "#            color=df.Cluster_color.values,\n",
        "#            #colorscale=\"Viridis\",\n",
        "#        ),\n",
        "#        line=dict(color=\"darkblue\", width=0.01),\n",
        "#        hovertext=df.protocol_description.values,\n",
        "#        hoverinfo='text'\n",
        "#    )\n",
        "#)\n",
        "#\n",
        "#fig.update_layout(\n",
        "#    title=\"3D Plotted Biotool OpenAI Ada-code embeddings (TriMAP)\",\n",
        "#    width=1000,\n",
        "#    height=1000,\n",
        "#    autosize=False,\n",
        "#    scene=dict(\n",
        "#        camera=dict(\n",
        "#            up=dict(x=0, y=0, z=1),\n",
        "#            eye=dict(\n",
        "#                x=0,\n",
        "#                y=1.0707,\n",
        "#                z=1,\n",
        "#            ),\n",
        "#        ),\n",
        "#        aspectratio=dict(x=1, y=1, z=0.7),\n",
        "#        aspectmode=\"manual\",\n",
        "#    ),\n",
        "#)\n",
        "#\n",
        "#fig.show()\n",
        "\n",
        "protocol_colors_list = []\n",
        "\n",
        "trace1 = go.Scatter3d(\n",
        "    x=Xe,\n",
        "    y=Ye,\n",
        "    z=Ze,\n",
        "    mode=\"lines\",\n",
        "    line=dict(color=\"rgb(125,125,125)\", width=0.001),\n",
        "    hoverinfo=\"none\",\n",
        ")\n",
        "\n",
        "trace2 = go.Scatter3d(\n",
        "    x=Xn,\n",
        "    y=Yn,\n",
        "    z=Zn,\n",
        "    mode=\"markers\",\n",
        "    name=\"actors\",\n",
        "    marker=dict(\n",
        "        symbol=\"circle\",\n",
        "        size=2,\n",
        "        color=df.language_colors.values,\n",
        "    ),\n",
        "    hovertext=df.hovertext.values,\n",
        "    hoverinfo=\"text\",\n",
        ")\n",
        "\n",
        "axis = dict(\n",
        "    showbackground=True,\n",
        "    showline=True,\n",
        "    zeroline=True,\n",
        "    title=\"\",\n",
        ")\n",
        "\n",
        "layout = go.Layout(\n",
        "    title=\"Network of 3D Plotted Biotool OpenAI Ada-code embeddings<br> created using TriMAP\",\n",
        "    width=1000,\n",
        "    height=1000,\n",
        "    showlegend=False,\n",
        "    scene=dict(\n",
        "        xaxis=dict(axis),\n",
        "        yaxis=dict(axis),\n",
        "        zaxis=dict(axis),\n",
        "    ),\n",
        "    margin=dict(t=100),\n",
        ")\n",
        "\n",
        "data = [trace1, trace2]\n",
        "fig = go.Figure(data=data, layout=layout)\n",
        "\n",
        "# Add drowdowns\n",
        "# button_layer_1_height = 1.08\n",
        "#button_layer_1_height = 1.12\n",
        "#button_layer_2_height = 1.065\n",
        "#\n",
        "#fig.update_layout(\n",
        "#    updatemenus=[\n",
        "#        dict(\n",
        "#            buttons=list([\n",
        "#                dict(\n",
        "#                    args=[\"colorscale\", \"Viridis\"],\n",
        "#                    label=\"Viridis\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#                dict(\n",
        "#                    args=[\"colorscale\", \"Cividis\"],\n",
        "#                    label=\"Cividis\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#                dict(\n",
        "#                    args=[\"colorscale\", \"Blues\"],\n",
        "#                    label=\"Blues\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#                dict(\n",
        "#                    args=[\"colorscale\", \"Greens\"],\n",
        "#                    label=\"Greens\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#            ]),\n",
        "#            type = \"buttons\",\n",
        "#            direction=\"right\",\n",
        "#            pad={\"r\": 10, \"t\": 10},\n",
        "#            showactive=True,\n",
        "#            x=0.1,\n",
        "#            xanchor=\"left\",\n",
        "#            y=button_layer_1_height,\n",
        "#            yanchor=\"top\"\n",
        "#        ),\n",
        "#        dict(\n",
        "#            buttons=list([\n",
        "#                dict(\n",
        "#                    args=[\"reversescale\", False],\n",
        "#                    label=\"False\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#                dict(\n",
        "#                    args=[\"reversescale\", True],\n",
        "#                    label=\"True\",\n",
        "#                    method=\"restyle\"\n",
        "#                )\n",
        "#            ]),\n",
        "#            type = \"buttons\",\n",
        "#            direction=\"right\",\n",
        "#            pad={\"r\": 10, \"t\": 10},\n",
        "#            showactive=True,\n",
        "#            x=0.13,\n",
        "#            xanchor=\"left\",\n",
        "#            y=button_layer_2_height,\n",
        "#            yanchor=\"top\"\n",
        "#        ),\n",
        "#        dict(\n",
        "#            buttons=list([\n",
        "#                dict(\n",
        "#                    args=[{\"contours.showlines\": False, \"type\": \"contour\"}],\n",
        "#                    label=\"Hide lines\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#                dict(\n",
        "#                    args=[{\"contours.showlines\": True, \"type\": \"contour\"}],\n",
        "#                    label=\"Show lines\",\n",
        "#                    method=\"restyle\"\n",
        "#                ),\n",
        "#            ]),\n",
        "#            type = \"buttons\",\n",
        "#            direction=\"right\",\n",
        "#            pad={\"r\": 10, \"t\": 10},\n",
        "#            showactive=True,\n",
        "#            x=0.5,\n",
        "#            xanchor=\"left\",\n",
        "#            y=button_layer_2_height,\n",
        "#            yanchor=\"top\"\n",
        "#        ),\n",
        "#    ]\n",
        "#)\n",
        "#\n",
        "#fig.update_layout(\n",
        "#    annotations=[\n",
        "#        dict(text=\"colorscale\", x=0, xref=\"paper\", y=1.1, yref=\"paper\",\n",
        "#                             align=\"left\", showarrow=False),\n",
        "#        dict(text=\"Reverse<br>Colorscale\", x=0, xref=\"paper\", y=1.06,\n",
        "#                             yref=\"paper\", showarrow=False),\n",
        "#        dict(text=\"Lines\", x=0.47, xref=\"paper\", y=1.045, yref=\"paper\",\n",
        "#                             showarrow=False)\n",
        "#    ])\n",
        "\n",
        "fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "c-xv-AkEAdLA"
      },
      "outputs": [],
      "source": [
        "fig.write_html(\"3D_embedding_exploration_tool.html\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nCgAaIBbifwL"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jzCpCPz-ifti"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}