notebooks/es_add_alias.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ElasticSearch alias and index update testing\n",
    "\n",
    "For testing out ElasticSearch commands before attaching them to django tasks.\n",
    "_Needs to be run from the django_app poetry environment._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get this working with django \n",
    "\n",
    "import os, sys\n",
    "import django\n",
    "PROJECTPATH = '.'\n",
    "sys.path.insert(0, PROJECTPATH)\n",
    "os.environ.setdefault(\"DJANGO_SETTINGS_MODULE\", \"redbox_app.settings\")\n",
    "os.environ[\"DJANGO_ALLOW_ASYNC_UNSAFE\"] = \"true\"\n",
    "os.chdir(PROJECTPATH)\n",
    "django.setup()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. General ES setup\n",
    "Getting the settings and the client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from redbox.models import Settings, get_settings\n",
    "\n",
    "env = Settings()\n",
    "\n",
    "es_client = env.elasticsearch_client()\n",
    "\n",
    "es_root_index = env.elastic_root_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "es_root_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Adding ES alias\n",
    "\n",
    "Adds an alias to the existing data.\n",
    "\n",
    "_Command will be in `django_app.redbox_app.redbox_core.management.commands.add_es_alias.py`_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "existing_index = f\"{env.elastic_root_index}-chunk\"\n",
    "\n",
    "if not es_client.indices.exists_alias(name=f\"{existing_index}-current\"):\n",
    "    es_client.indices.put_alias(index=existing_index, name=f\"{existing_index}-current\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Reingesting files\n",
    "This adds a new index for these files.\n",
    "\n",
    "_Command will be in `django_app/redbox_app/redbox_core/management/commands/reingest_files.py`_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import logging\n",
    "\n",
    "from django_q.tasks import async_task\n",
    "\n",
    "from redbox.models import Settings, get_settings\n",
    "from redbox_app.redbox_core.models import INACTIVE_STATUSES, File\n",
    "from redbox_app.worker import ingest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "default_index = f\"{env.elastic_root_index}-chunk\"\n",
    "new_index = f\"{default_index}-{datetime.datetime.now(tz=datetime.UTC).strftime('%y%m%d%H%M')}\"\n",
    "\n",
    "for file in File.objects.exclude(status__in=INACTIVE_STATUSES):\n",
    "    # logger.debug(\"Reingesting file object %s\", file)\n",
    "    print(\"Reingesting file object %s\", file)\n",
    "    async_task(\n",
    "        ingest, file.id, new_index, task_name=file.original_file.name, group=\"re-ingest\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Change which index is aliased\n",
    "\n",
    "This moves the alias to the newly-created index. It will require finding the index created as one of the args in a reingest-file task in `[domain]/admin/django_q/success/`.\n",
    "\n",
    "_Command will be in `django_app/redbox_app/redbox_core/management/commands/change_es_aliased_index.py`_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_index = \"redbox-data-chunk-2410011041\"\n",
    "alias = f\"{env.elastic_root_index}-chunk-current\"\n",
    "\n",
    "try:\n",
    "    response = es_client.indices.get_alias(name=alias)\n",
    "    indices_to_remove = list(response.keys())\n",
    "except ElasticsearchException as e:\n",
    "    logger.exception(\"Error fetching alias\", exc_info=e)\n",
    "\n",
    "# logger.info(f\"indices to remove: {indices_to_remove}\")\n",
    "print(f\"indices to remove: {indices_to_remove}\")\n",
    "actions = [{\"remove\": {\"index\": index, \"alias\": alias}} for index in indices_to_remove]\n",
    "actions.append({\"add\": {\"index\": new_index, \"alias\": alias}})\n",
    "print(actions)\n",
    "\n",
    "es_client.indices.update_aliases(body={\"actions\": actions})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Remove old data\n",
    "\n",
    "This clears out the data from the other indexes we no longer need.\n",
    "\n",
    "_Command will be in `django_app/redbox_app/redbox_core/management/commands/delete_es_indices.py`_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_chunk_indices():\n",
    "    try:\n",
    "        # Get all indices\n",
    "        indices = es_client.cat.indices(format=\"json\")\n",
    "        # Filter indices that contain '-chunk'\n",
    "        return [index[\"index\"] for index in indices if \"-chunk\" in index[\"index\"]]\n",
    "    except Exception as e:\n",
    "        # logger.exception(\"Error fetching indices\", exc_info=e)\n",
    "        print(\"Error fetching indices\")\n",
    "\n",
    "for index in list_chunk_indices():\n",
    "    if index != new_index:\n",
    "        print(f\"deleting {index}\")\n",
    "        es_client.indices.delete(index=index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}