pyspark_k8s_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5faaaba9-70e0-4edd-a126-3994f3d9fb48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "51403f42-f110-4ba7-9b2e-1021fff71486",
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9b643160-9d0a-4a18-9890-8d9c9cfcae6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkConf, SparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b6fc7ee8-44cc-4507-9ebf-cc503694600a",
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = (SparkConf().setMaster(\"k8s://https://192.168.49.2:8443\") # Your master address name\n",
    "        .set(\"spark.kubernetes.container.image\", \"pyspark:latest\") # Spark image name\n",
    "        .set(\"spark.driver.port\", \"2222\") # Needs to match svc\n",
    "        .set(\"spark.driver.blockManager.port\", \"7777\")\n",
    "        .set(\"spark.driver.host\", \"driver-service.jupyterhub.svc.cluster.local\") # Needs to match svc\n",
    "        .set(\"spark.driver.bindAddress\", \"0.0.0.0\")\n",
    "        .set(\"spark.kubernetes.namespace\", \"spark\")\n",
    "        .set(\"spark.kubernetes.authenticate.driver.serviceAccountName\", \"spark\")\n",
    "        .set(\"spark.kubernetes.authenticate.serviceAccountName\", \"spark\")\n",
    "\n",
    "        .set(\"spark.executor.instances\", \"1\")\n",
    "        .set(\"spark.kubernetes.container.image.pullPolicy\", \"IfNotPresent\")\n",
    "       .set(\"spark.app.name\", \"YOUR APP NAME\"))\n",
    "       \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c4828e16-b301-41a3-9d9b-8811725d7b6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/home/jovyan/.local/lib/python3.9/site-packages/pyspark/jars/spark-unsafe_2.12-3.2.2.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "22/10/16 17:46:36 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    }
   ],
   "source": [
    "sc = SparkContext(conf=conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b679b2d8-eb6a-4515-9a07-e5e5371c9615",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = sc.parallelize(range(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "49c0786f-e448-4275-b23e-357d22c0034b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Approximate sum: 45.0\n"
     ]
    }
   ],
   "source": [
    "# Calculate the approximate sum of values in the dataset\n",
    "r = t.sumApprox(3)\n",
    "print('Approximate sum: %s' % r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4f0176d-a83f-4242-8fe4-a8bfb38e2bc5",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}