domain_datasets_generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "28afeacb-a6f4-4421-9600-a0081926e6c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark.sql.functions as F\n",
    "from pyspark.sql.types import ArrayType, StringType, BooleanType, IntegerType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "63e2a207-cd3d-4299-b520-73b8b69e845a",
   "metadata": {},
   "outputs": [],
   "source": [
    "path2openAIRE = \"/export/ml4ds/IntelComp/Datalake/openaire/20230328/parquet/\"\n",
    "path2S2 = \"/export/ml4ds/IntelComp/Datalake/semanticscholar/20230418/parquet/\"\n",
    "path2PATSTAT = \"/export/ml4ds/IntelComp/Datalake/patstat/2023_Spring/parquet/\"\n",
    "path2cordis = \"/export/ml4ds/IntelComp/Datalake/CORDIS/20230823/parquet/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c5ebbb8-587e-4e1c-b26d-1ca3c3777531",
   "metadata": {},
   "source": [
    "# 1. Cancer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b528c25-be15-40e8-a4b7-9e1b1fa6e586",
   "metadata": {},
   "source": [
    "## 1.1. Definition of the target datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "137907a8-6c3b-4d74-8281-bbcc02bd9a2d",
   "metadata": {},
   "source": [
    "### 1.1.1. Publications\n",
    "\n",
    "HCERES has provided a closed list of publications for inclusion in the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36fdf45b-f892-4c7f-8db4-63022a1bb2d6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of provided publications in the cancer domain: 2438253\n",
      "-RECORD 0--------\n",
      " PMID | 10725091 \n",
      "-RECORD 1--------\n",
      " PMID | 10725105 \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cancer_PMID_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PMID_Cancer.parquet\"\n",
    "cancer_PMID = spark.read.parquet(cancer_PMID_file).cache()\n",
    "print('Number of provided publications in the cancer domain:', cancer_PMID.count())\n",
    "cancer_PMID.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b67184d5-08ef-4eb1-88f7-9e6abd1b60a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cancer publications in Semantic Scholar: 2436965\n",
      "Number of cancer publications in Semantic Scholar with non null abstract: 1522138\n"
     ]
    }
   ],
   "source": [
    "# Matching with Semantic Scholar publications\n",
    "\n",
    "papers_S2 = spark.read.parquet(path2S2 + \"papers.parquet\")\n",
    "col_to_keep = [\"id\", \"pmid\", \"doi\", \"year\", \"title\", \"paperAbstract\"]\n",
    "papers_S2 = papers_S2.select(*col_to_keep).cache()\n",
    "\n",
    "cancer_S2 = cancer_PMID.join(papers_S2, cancer_PMID.PMID == papers_S2.pmid, how='inner').drop(\"PMID\").cache()\n",
    "print('Number of cancer publications in Semantic Scholar:', cancer_S2.count())\n",
    "print('Number of cancer publications in Semantic Scholar with non null abstract:', cancer_S2.filter(F.col(\"paperAbstract\").isNotNull()).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ee2ab9e8-5c2e-4430-a992-8d03073d7658",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_S2_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/S2_Cancer.parquet\"\n",
    "cancer_S2.write.parquet(cancer_S2_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0cdf437e-2f7a-4199-a73a-9caccd5a4eb5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/08/29 02:32:50 WARN SharedInMemoryCache: Evicting cached table partition metadata from memory due to size constraints (spark.sql.hive.filesourcePartitionFileCacheSize = 262144000 bytes). This may impact query planning performance.\n",
      "[Stage 10:==================================================>  (190 + 10) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cancer publications in OpenAire: 2393228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Matching with OpenAIRE publications\n",
    "\n",
    "result_instance_pid = spark.read.parquet(path2openAIRE + \"result_instance_pid\")\n",
    "result_pmid = result_instance_pid.filter(result_instance_pid[\"type\"] == \"PubMed ID\")\n",
    "col_to_keep = [\"result\", \"pid\"]\n",
    "result_pmid = result_pmid.select(*col_to_keep).withColumnRenamed(\"result\",\"id\").withColumnRenamed(\"pid\",\"pmid\").dropDuplicates()\n",
    "\n",
    "cancer_OA = cancer_PMID.join(result_pmid, cancer_PMID.PMID == result_pmid.pmid, how=\"inner\").drop(\"PMID\").cache()\n",
    "print('Number of cancer publications in OpenAire:', cancer_OA.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3a874a76-2fde-4542-91ad-0b630e5512e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_titles = spark.read.parquet(path2openAIRE + \"result_titles\").dropDuplicates(subset=[\"id\"])\n",
    "cancer_OA = cancer_OA.join(result_titles, on=\"id\", how=\"left\").drop(result_titles.id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "21f3cc8d-4a89-41ea-90d0-dcfb60da5900",
   "metadata": {},
   "outputs": [],
   "source": [
    "result_description = spark.read.parquet(path2openAIRE + \"result_description\").dropDuplicates(subset=[\"id\"])\n",
    "cancer_OA = cancer_OA.join(result_description, on=\"id\", how=\"left\").drop(result_description.id).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b61e71ee-5f96-416d-ae76-8d106985db54",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 24:=====================================================>(199 + 1) / 200]]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cancer publications in OpenAIRE with non null abstract: 2329760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "print('Number of cancer publications in OpenAIRE with non null abstract:', cancer_OA.filter(F.col(\"description\").isNotNull()).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a25e012-b625-490d-b060-e20cc9051b8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_OA_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/OA_Cancer.parquet\"\n",
    "cancer_OA.write.parquet(cancer_OA_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4fe14e3e-69eb-4021-be7c-c1e495ecf1b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0-------------------------------------------------------------------------------------------------------------------------------\n",
      " id          | doi_dedup___::000e32b41396c454e9d665a32c959feb                                                                           \n",
      " title       | Long-Term Laryngeal Allograft Survival Using Low-Dose Everolimus                                                         \n",
      " description | Abstract   Objective  The purpose of this study was to explore the mechanism and utility of everolimus as a single-ag... \n",
      "-RECORD 1-------------------------------------------------------------------------------------------------------------------------------\n",
      " id          | doi_dedup___::000e600156867c15e1ab7adbabd38b3f                                                                           \n",
      " title       | The cost-saving effect of centralized histological reviews with soft tissue and visceral sarcomas, GIST, and desmoid ... \n",
      " description | Objective   This study examined the types of discordance occurring in the diagnosis of soft tissue and visceral sarco... \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cancer_OA.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38e8264b-56d4-4112-9086-4dce80d5a5a7",
   "metadata": {},
   "source": [
    "### 1.1.2. Patents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a12c7bad-f44f-4b4f-a5db-f9fc7759ce47",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/08/30 11:25:32 WARN ObjectStore: Failed to get database parquet, returning NoSuchObjectException\n",
      "[Stage 23:=====================================================>(399 + 1) / 400]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of patents with abstract in English language: 75348760\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      "\n",
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10000172                                                                                                                 \n",
      " docdb_family_id      | 5871993                                                                                                                  \n",
      " appln_title          | 2-Hydroxy-3-cyano-pyridone-(6) derivs - coupling compts. for the prepn. of azo dyes                                      \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The novel cpds. are of formula (I): (where R is (substd.)alkyl-, alkylene-, aralkyl-, cycloalkyl- or phenyl). They ar... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1973                                                                                                                     \n",
      " earliest_filing_year | 1973                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10000989                                                                                                                 \n",
      " docdb_family_id      | 5872463                                                                                                                  \n",
      " appln_title          | Sowing partially germinated seeds - by mixing seeds with fillers and transferring fillers and germinated seeds into soil \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | Seeds are sown in a manner reducing the germination time and in a weed free environment by mixing the seeds with fill... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1973                                                                                                                     \n",
      " earliest_filing_year | 1973                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "query = \"select appln_id, docdb_family_id, appln_title, appln_title_lg, appln_abstract, appln_abstract_lg, appln_filing_year, earliest_filing_year, \"\n",
    "query += \"granted, appln_auth, receiving_office, ipr_type from parquet.`\" + path2PATSTAT + \"/patstat_appln.parquet`\"\n",
    "df_patents = spark.sql(query)\n",
    "\n",
    "df_patents = df_patents.filter(F.col(\"appln_abstract\").isNotNull()).filter(df_patents.appln_abstract_lg==\"en\").cache()\n",
    "\n",
    "print('Number of patents with abstract in English language:', df_patents.count())\n",
    "df_patents.printSchema()\n",
    "df_patents.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a68a4db-b118-43d4-87a4-cda436403382",
   "metadata": {},
   "source": [
    "#### 1.1.2.1. Filtering according to CPC\n",
    "\n",
    "HCERES has provided a list of CPC codes related to cancer. We will use them to retrieve a first series of patents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f7048909-db66-4088-846c-2616ef2a45d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/08/30 11:13:59 WARN ObjectStore: Failed to get database parquet, returning NoSuchObjectException\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cpc entries: 372088875\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- cpc_class_symbol: string (nullable = true)\n",
      "\n",
      "-RECORD 0-----------------------\n",
      " appln_id         | 26591291    \n",
      " cpc_class_symbol | C22C  18/04 \n",
      "-RECORD 1-----------------------\n",
      " appln_id         | 26591291    \n",
      " cpc_class_symbol | H01M   4/42 \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "query = \"select appln_id, cpc_class_symbol from parquet.`\" + path2PATSTAT + \"tls224.parquet`\"\n",
    "df_cpc = spark.sql(query)\n",
    "\n",
    "print('Number of cpc entries:', df_cpc.count())\n",
    "df_cpc.printSchema()\n",
    "df_cpc.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5f3a640e-3914-4c07-9aa6-8a6366ddffa5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cpc entries: 460673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of applications: 293084\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- cpc_class_symbol: array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 19:=====================================================>  (64 + 3) / 67]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0----------------------------\n",
      " appln_id         | 10007045         \n",
      " cpc_class_symbol | [C12N   5/0693]  \n",
      "-RECORD 1----------------------------\n",
      " appln_id         | 1001853          \n",
      " cpc_class_symbol | [G01N  33/57434] \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_cpcs = [\"C12Q1/6886\",\"G01N33/574\",\"A61K39/0011\",\"G01N33/5011\",\"G01N33/57484\",\"C07K16/30\",\"G01N33/57492\",\"A61L2300/416\",\"G01N33/57434\",\"G01N33/53\",\"G01N33/57423\",\"G01N33/57419\",\"G01N33/57415\",\"G01N33/57407\",\"G01N33/57488\",\"C12N15/1135\",\"A61K47/48569\",\"G01N33/57438\",\"G06T2207/30096\",\"G01N33/57496\",\"G01N33/57426\",\"C07K16/3053\",\"C07K16/3069\",\"G01N33/57449\",\"C12N5/0693\",\"A23V2200/308\",\"C07K16/3015\",\"A61K47/48584\",\"C07K16/3007\",\"C07K16/3046\",\"C07K16/3061\",\"G01N33/57446\",\"A61K2039/5152\",\"A61K2039/585\",\"C07K16/303\",\"A61K51/1045\",\"G01N2800/7028\",\"G01N33/5743\",\"A61K35/13\",\"A61K47/48638\",\"C07K16/3023\",\"G01N33/5017\",\"G01N33/5748\",\"A61K35/768\",\"A61K47/4863\",\"G01N2333/82\",\"G01N33/57411\",\"C07K16/3076\",\"A61K47/48615\",\"G01N33/57442\",\"Y10S436/813\",\"A61K47/48576\",\"A61K51/1072\",\"C07K16/3038\",\"A61K51/1057\",\"A61K51/1048\",\"C12N5/0093\",\"A61K47/48607\",\"Y10S977/911\",\"A61K38/1764\",\"G01N33/57476\",\"G01N33/57469\",\"A61K51/106\",\"A61K45/05\",\"G01N2033/57403\",\"G01N2033/57461\",\"G01N2033/57465\",\"G01N2033/57453\",\"G01N2033/57457\"]\n",
    "CPC_match_UDF = F.udf(lambda x: x.replace(' ', '') in cancer_cpcs, BooleanType())\n",
    "\n",
    "# df_FOS.withColumn(\"FOS_source\", myUDF(F.col(\"S2fieldsofstudy\"))) \\\n",
    "#   .show(n=2, truncate=120, vertical=True)\n",
    "\n",
    "df_cpc_cancer = df_cpc.withColumn(\"isCancer\", CPC_match_UDF(F.col(\"cpc_class_symbol\"))).filter(\"isCancer\").drop(\"isCancer\").cache()\n",
    "\n",
    "print('Number of cpc entries:', df_cpc_cancer.count())\n",
    "\n",
    "df_appln_cancer = df_cpc_cancer.groupBy(\"appln_id\").agg(F.collect_list(\"cpc_class_symbol\").alias(\"cpc_class_symbol\"))\n",
    "\n",
    "print('Number of applications:', df_appln_cancer.count())\n",
    "df_appln_cancer.printSchema()\n",
    "df_appln_cancer.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7832cba9-36af-4811-a6bc-38ab45a57c91",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected CPC: 209547\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      " |-- cpc_class_symbol: array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 37:>                                                         (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 128357                                                                                                                   \n",
      " docdb_family_id      | 37963490                                                                                                                 \n",
      " appln_title          | CANCER IMAGING AND TREATMENT                                                                                             \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | A compound, or a pharmaceutically acceptable salt or ester thereof, comprises a ligand for the chemokine receptor CXC... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2007                                                                                                                     \n",
      " earliest_filing_year | 2006                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | EP                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      " cpc_class_symbol     | [G01N  33/574]                                                                                                           \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 13004786                                                                                                                 \n",
      " docdb_family_id      | 7702641                                                                                                                  \n",
      " appln_title          | Implant e.g. a stent, comprises a decomposable substance which allows contact between the cell proliferation inhibito... \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | An implant (1), with main structure (2) comprises a first substance (4) inhibiting cell proliferation and a second de... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2001                                                                                                                     \n",
      " earliest_filing_year | 2001                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      " cpc_class_symbol     | [A61L2300/416]                                                                                                           \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "selected_CPC = df_patents.join(df_appln_cancer, df_patents.appln_id == df_appln_cancer.appln_id, \"inner\").drop(df_appln_cancer.appln_id)\n",
    "print('Selected CPC:', selected_CPC.count())\n",
    "selected_CPC.printSchema()\n",
    "selected_CPC.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3b67cdd3-9b03-4cc2-96c7-504f589a886b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_CPC_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_CPC_Cancer.parquet\"\n",
    "\n",
    "selected_CPC.repartition(40).write.parquet(cancer_CPC_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf084eb6-75b4-4678-b4b3-501c6ae9094c",
   "metadata": {},
   "source": [
    "#### 1.1.2.2. Filtering according to Keywords\n",
    "\n",
    "Additionally, we have a list of 123 keywords provided by HCERES that can be used for the selection of patents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "120b00fa-46f7-4974-8d22-2d6f5f49feea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of keywords: 123\n"
     ]
    }
   ],
   "source": [
    "with open(\"/export/clusterdata/jarenas/Datasets/patstat/2023_Spring/Cancer_keywords_HCERES_patents.txt\", \"r\") as fin:\n",
    "    Cancer_kwds = [el.strip() for el in fin.readlines()]\n",
    "    Cancer_kwds = [el for el in Cancer_kwds if len(el)]\n",
    "print('Number of keywords:', len(Cancer_kwds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "51a98b99-069d-4ee2-b0cf-01244ddc8ea1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 47:=====================================================>(399 + 1) / 400]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in Cancer dataset: 580280\n",
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 108231                                                                                                                   \n",
      " docdb_family_id      | 40263006                                                                                                                 \n",
      " appln_title          | RNAI-RELATED INHIBITION OF TNFA SIGNALING PATHWAY FOR TREATMENT OF OCULAR ANGIOGENESIS                                   \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | RNA interference is provided for inhibition of tumor necrosis factor (TNF ) by silencing TNF cell surface receptor TN... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2008                                                                                                                     \n",
      " earliest_filing_year | 2007                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | WO                                                                                                                       \n",
      " receiving_office     | US                                                                                                                       \n",
      " ipr_type             | PI                                                                                                                       \n",
      " raw_text             | RNAI-RELATED INHIBITION OF TNFA SIGNALING PATHWAY FOR TREATMENT OF OCULAR ANGIOGENESIS RNA interference is provided f... \n",
      " Kwd_count            | 1                                                                                                                        \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 11188559                                                                                                                 \n",
      " docdb_family_id      | 6429632                                                                                                                  \n",
      " appln_title          | Therapeutic radiation equipment using lamp as light source with control system - uses optical filter components contg... \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The radiation system uses the working principle of photodynamic therapy (PDT), for the radiation treatment of e.g. tu... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1991                                                                                                                     \n",
      " earliest_filing_year | 1991                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      " raw_text             | Therapeutic radiation equipment using lamp as light source with control system - uses optical filter components contg... \n",
      " Kwd_count            | 2                                                                                                                        \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "def count_kwds(x):\n",
    "    try:\n",
    "        reps = [x.lower().count(k) for k in Cancer_kwds]\n",
    "        return sum(reps)\n",
    "    except:\n",
    "        return 0\n",
    "\n",
    "count_kwds_UDF = F.udf(count_kwds, IntegerType())\n",
    "\n",
    "Cancer_patents = df_patents.withColumn('raw_text', \n",
    "                    F.concat(F.col('appln_title'),F.lit(' '), F.col('appln_abstract')))\n",
    "\n",
    "Cancer_patents = Cancer_patents.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "Cancer_patents = Cancer_patents.filter(Cancer_patents.Kwd_count > 0).cache()\n",
    "\n",
    "print('Number of documents in Cancer dataset:', Cancer_patents.count())\n",
    "Cancer_patents.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9f18a1b4-3aae-45a7-8aa3-977e5e804f29",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_Kwds_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_Kwds_Cancer.parquet\"\n",
    "\n",
    "Cancer_patents.drop(\"raw_text\").drop(\"Kwd_count\").repartition(40).write.parquet(cancer_Kwds_file, mode=\"overwrite\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d077303f-5d33-4b3f-954b-9001f65b75d4",
   "metadata": {},
   "source": [
    "#### 1.1.2.3. Filtering according to citations related to Cancer\n",
    "\n",
    "A last strategy is to select patents that include citations to papers which have been selected by HCERES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e0664543-3f93-4361-bf03-6bd4e7a8ac81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Cancer publications in S2: 2436965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 19161039                                                                                                                 \n",
      " doi           | 10.1002/(SICI)1098-2264(200002)27:2<162::AID-GCC7>3.0.CO;2-9                                                             \n",
      " year          | 2000                                                                                                                     \n",
      " title         | Molecular genetic alterations in carcinoma ex‐pleomorphic adenoma: A putative progression model?                         \n",
      " paperAbstract | To determine the genetic changes associated with the development of carcinoma ex‐pleomorphic adenoma (Ca Ex‐PA), we a... \n",
      "-RECORD 1---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 46428546                                                                                                                 \n",
      " doi           | 10.1016/S0277-9536(99)00256-7                                                                                            \n",
      " year          | 2000                                                                                                                     \n",
      " title         | Rethinking patient satisfaction: patient experiences of an open access flexible sigmoidoscopy service.                   \n",
      " paperAbstract | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n",
      "-RECORD 0-----------------------------------------------------------\n",
      " doi | 10.1002/(SICI)1098-2264(200002)27:2<162::AID-GCC7>3.0.CO;2-9 \n",
      "-RECORD 1-----------------------------------------------------------\n",
      " doi | 10.1016/S0277-9536(99)00256-7                                \n",
      "-RECORD 2-----------------------------------------------------------\n",
      " doi | 10.1006/EXCR.1999.4736                                       \n",
      "-RECORD 3-----------------------------------------------------------\n",
      " doi | 10.1128/JVI.74.2.899-913.2000                                \n",
      "-RECORD 4-----------------------------------------------------------\n",
      " doi | 10.1016/S1383-5718(99)00174-6                                \n",
      "only showing top 5 rows\n",
      "\n",
      "Number of Cancer publications with doi: 2310347\n"
     ]
    }
   ],
   "source": [
    "cancer_S2 = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/S2_Cancer.parquet')\n",
    "\n",
    "print('Number of Cancer publications in S2:', cancer_S2.count())\n",
    "cancer_S2.show(n=2, truncate=120, vertical=True)\n",
    "\n",
    "cancer_dois = cancer_S2.filter(F.col(\"doi\").isNotNull()).select(\"doi\")\n",
    "cancer_dois.show(n=5, truncate=120, vertical=True)\n",
    "print('Number of Cancer publications with doi:', cancer_dois.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4023d8eb-e3cd-4ee4-aa14-d67230f511c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/08/30 11:58:08 WARN ObjectStore: Failed to get database parquet, returning NoSuchObjectException\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "71450314"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"select appln_id, npl_doi as doi from parquet.`\" + path2PATSTAT + \"npl_citations.parquet`\"\n",
    "citations = spark.sql(query)\n",
    "citations.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3c2ff384-44b7-46fa-8462-ec9079e1a428",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 95:======================================================> (49 + 1) / 50]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1938151\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "citations = citations.filter(F.col(\"doi\").isNotNull())\n",
    "print(citations.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c538c7e4-92e2-4e5d-b76a-eec313f98ded",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 102:===============================================>       (38 + 4) / 44]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "159038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "intersect = citations.join(cancer_dois, citations.doi==cancer_dois.doi, \"inner\")\n",
    "print(intersect.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b81956d2-2b6a-40e0-9d32-ab28183b1650",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "77789"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "patents_cancer_npl = intersect.select(\"appln_id\").distinct()\n",
    "patents_cancer_npl.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "cc512774-a6a1-4728-ae2d-88a560b64573",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 139:==================================================>    (41 + 4) / 45]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0-------------\n",
      " appln_id | 407079753 \n",
      "-RECORD 1-------------\n",
      " appln_id | 539836071 \n",
      "-RECORD 2-------------\n",
      " appln_id | 538573027 \n",
      "-RECORD 3-------------\n",
      " appln_id | 522657239 \n",
      "-RECORD 4-------------\n",
      " appln_id | 479356246 \n",
      "only showing top 5 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "patents_cancer_npl.show(n=5, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "8bce4705-70c2-4999-934c-51d186269db4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected publis: 75128\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 16257674                                                                                                                 \n",
      " docdb_family_id      | 40262947                                                                                                                 \n",
      " appln_title          | PREPRIMITIVE STREAK AND MESENDODERM CELLS                                                                                \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | This disclosure relates to compositions comprising human preprimitive streak cells and/or human mesendoderm cells as ... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2005                                                                                                                     \n",
      " earliest_filing_year | 2004                                                                                                                     \n",
      " granted              | Y                                                                                                                        \n",
      " appln_auth           | EP                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 171969                                                                                                                   \n",
      " docdb_family_id      | 39033521                                                                                                                 \n",
      " appln_title          | INHIBITORS OF PYRUVATE KINASE AND METHODS OF TREATING DISEASE                                                            \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The invention provides pharmaceutical compositions, kits, and methods of treating cancer, diabetes, obesity, autoimmu... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2007                                                                                                                     \n",
      " earliest_filing_year | 2006                                                                                                                     \n",
      " granted              | Y                                                                                                                        \n",
      " appln_auth           | EP                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "selected_publis = df_patents.join(patents_cancer_npl, on = \"appln_id\", how = \"inner\").drop(patents_cancer_npl.appln_id)\n",
    "print('Selected publis:', selected_publis.count())\n",
    "selected_publis.printSchema()\n",
    "selected_publis.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7165e3d7-af5c-4924-9e23-3a609d4a30d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_npl_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_npl_Cancer.parquet\"\n",
    "\n",
    "selected_publis.repartition(40).write.parquet(cancer_npl_file, mode=\"overwrite\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc1c3559-9fff-4eb6-9327-151a8b961457",
   "metadata": {},
   "source": [
    "#### 1.1.2.4. Calculating the intersection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0f1cf245-c86b-48df-842e-5662d5577950",
   "metadata": {},
   "outputs": [],
   "source": [
    "cancer_CPC = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_CPC_Cancer.parquet').drop(\"cpc_class_symbol\")\n",
    "cancer_Kwds = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_Kwds_Cancer.parquet')\n",
    "cancer_npl = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_npl_Cancer.parquet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ab898eb8-7389-400a-9b98-ea00c37de7eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of intersection: 131369\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 11246697                                                                                                                 \n",
      " docdb_family_id      | 6466484                                                                                                                  \n",
      " appln_title          | Isolation of circulating cancer cells from bloodstream                                                                   \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | Method (A) for isolating and culturing transformed cells circulating in the bloodstream of an individual comprises tr... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1992                                                                                                                     \n",
      " earliest_filing_year | 1992                                                                                                                     \n",
      " granted              | Y                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 11293487                                                                                                                 \n",
      " docdb_family_id      | 6516627                                                                                                                  \n",
      " appln_title          | Analysis and detection of tumours using variable CD44 exons,                                                             \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | Methods for the diagnosis and/or analysis of tumours by detecting at least one variable exon of the CD44 gene or corr... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1994                                                                                                                     \n",
      " earliest_filing_year | 1994                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_aux = cancer_Kwds.select(\"appln_id\")\n",
    "cancer_intersection = cancer_CPC.join(df_aux, on = \"appln_id\", how = \"inner\").drop(df_aux.appln_id)\n",
    "# df_aux = cancer_npl.select(\"appln_id\")\n",
    "# cancer_intersection = cancer_CPC.join(df_aux, on = \"appln_id\", how = \"inner\").drop(df_aux.appln_id)\n",
    "\n",
    "print('Size of intersection:', cancer_intersection.count())\n",
    "cancer_intersection.printSchema()\n",
    "cancer_intersection.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "81bedd8c-0dae-4081-9699-e7fa8a69f748",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_intersection_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_intersection_Cancer.parquet\"\n",
    "\n",
    "cancer_intersection.repartition(40).write.parquet(cancer_intersection_file, mode=\"overwrite\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88e8509b-a403-4e3d-b04d-ffdca28e66bd",
   "metadata": {},
   "source": [
    "#### 1.1.2.5 Calculating the union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "54069b9d-eaa8-44e0-9840-95c3fa9ba6ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of union: 698966\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 255:>                                                        (0 + 1) / 1]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10051360                                                                                                                 \n",
      " docdb_family_id      | 5897991                                                                                                                  \n",
      " appln_title          | Anti-cancer medicaments contg. amygdalin - causing tumour regression, preventing further tumour growth and having ana... \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | A novel medicament for treating cancer contains amygdalin (I) as active ingredient, opt in combination with thiamine ... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1973                                                                                                                     \n",
      " earliest_filing_year | 1973                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 100584                                                                                                                   \n",
      " docdb_family_id      | 39631112                                                                                                                 \n",
      " appln_title          | DNAI - LIPOSOMES                                                                                                         \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The invention provides oligonucleotide compositions and methods for the treatment of cancer.                             \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 2008                                                                                                                     \n",
      " earliest_filing_year | 2007                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | WO                                                                                                                       \n",
      " receiving_office     | US                                                                                                                       \n",
      " ipr_type             | PI                                                                                                                       \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_all = cancer_CPC.union(cancer_Kwds).union(cancer_npl).dropDuplicates([\"appln_id\"])\n",
    "\n",
    "print('Size of union:', cancer_all.count())\n",
    "cancer_all.printSchema()\n",
    "cancer_all.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d885d505-5ae6-434e-99cf-53ade3955bca",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_all_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/PATSTAT_all_Cancer.parquet\"\n",
    "\n",
    "cancer_all.repartition(40).write.parquet(cancer_all_file, mode=\"overwrite\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4802e8dd-7155-4b8e-a84b-d3f946041097",
   "metadata": {},
   "source": [
    "### 1.1.3. CORDIS projects"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb9aa87e-e8c9-4cc2-bf00-e52e465e32d0",
   "metadata": {},
   "source": [
    "#### 1.1.3.1. Filtering according to EuroSciVoc\n",
    "\n",
    "A first selection of projects is based on the EuroSciVoc codes. All projects under /21/39/225 (oncology) should be selected as relevant projects for the domain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d44fae6b-31b6-4fa2-8996-5394e59111a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects: 69613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of cancer projects according to EuroSciVoc: 4020\n",
      "root\n",
      " |-- projectID: long (nullable = true)\n",
      " |-- acronym: string (nullable = true)\n",
      " |-- status: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- startDate: timestamp (nullable = true)\n",
      " |-- endDate: timestamp (nullable = true)\n",
      " |-- totalCost: string (nullable = true)\n",
      " |-- ecMaxContribution: string (nullable = true)\n",
      " |-- ecSignatureDate: timestamp (nullable = true)\n",
      " |-- frameworkProgramme: string (nullable = true)\n",
      " |-- masterCall: string (nullable = true)\n",
      " |-- subCall: string (nullable = true)\n",
      " |-- fundingScheme: string (nullable = true)\n",
      " |-- nature: string (nullable = true)\n",
      " |-- objective: string (nullable = true)\n",
      " |-- contentUpdateDate: timestamp (nullable = true)\n",
      " |-- rcn: long (nullable = true)\n",
      " |-- grantDoi: string (nullable = true)\n",
      " |-- topic: string (nullable = true)\n",
      " |-- topic_title: string (nullable = true)\n",
      " |-- countryContr: string (nullable = true)\n",
      " |-- orgContr: string (nullable = true)\n",
      " |-- coordinatorCountry: string (nullable = true)\n",
      " |-- coordinatorOrg: double (nullable = true)\n",
      " |-- euroSciVocCode: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- publicationID: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- patentID: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      "\n",
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 200620                                                                                                                   \n",
      " acronym            | CANCERDIP                                                                                                                \n",
      " status             | CLO                                                                                                                      \n",
      " title              | The use of Methylated DNA Immunoprecipitation MeDIP in cancer for better clinical management                             \n",
      " startDate          | 2008-01-01 00:00:00                                                                                                      \n",
      " endDate            | 2011-06-30 00:00:00                                                                                                      \n",
      " totalCost          | 3922042.97                                                                                                               \n",
      " ecMaxContribution  | 2999994.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-HEALTH-2007-A                                                                                                        \n",
      " fundingScheme      | CP-FP                                                                                                                    \n",
      " nature             | null                                                                                                                     \n",
      " objective          | Aberrant DNA methylation is the most common molecular lesion of the cancer cell. Neither gene mutation nor cytogeneti... \n",
      " contentUpdateDate  | 2019-07-16 09:49:00                                                                                                      \n",
      " rcn                | 88147                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | HEALTH-2007-2.4.1-3                                                                                                      \n",
      " topic_title        | Genomic instability and genomic alterations in pre-cancerous lesions and/or cancer                                       \n",
      " countryContr       | BE|915498.0 DE|299999.0 ES|825001.0 IT|299998.0 NL|659498.0                                                              \n",
      " orgContr           | 999547850.0|285657.69 999992110.0|659498.0 999755139.0|265499.0 999986290.0|649999.0 999848356.0|299998.0 999990267.0... \n",
      " coordinatorCountry | ES                                                                                                                       \n",
      " coordinatorOrg     | 9.99600618E8                                                                                                             \n",
      " euroSciVocCode     | [1015, 1017, 225, 1025, 44114657]                                                                                        \n",
      " publicationID      | [B61F89E82970C5F6EB084699CAA1D528C01FF9C6, 7221DC3EAE837A94075854E425AA0E4156E9C976, E97FC7F0213371ED4C52623199D50476... \n",
      " patentID           | null                                                                                                                     \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 241344                                                                                                                   \n",
      " acronym            | MORPHOSTASIS                                                                                                             \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Morphostasis of the intestinal mucosa and it's deregulation in cancer and inflammation                                   \n",
      " startDate          | 2009-10-01 00:00:00                                                                                                      \n",
      " endDate            | 2014-09-30 00:00:00                                                                                                      \n",
      " totalCost          | 1524462.0                                                                                                                \n",
      " ecMaxContribution  | 1524462.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | ERC-2009-StG                                                                                                             \n",
      " fundingScheme      | ERC-SG                                                                                                                   \n",
      " nature             | null                                                                                                                     \n",
      " objective          | Stem cells at the base of the intestinal crypts are in a dynamic equilibrium with their differentiated derivatives. H... \n",
      " contentUpdateDate  | 2019-08-01 17:00:37                                                                                                      \n",
      " rcn                | 92293                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | ERC-SG-LS4                                                                                                               \n",
      " topic_title        | ERC Starting Grant - Physiology, Pathophysiology and Endocrinology                                                       \n",
      " countryContr       | NL|1524462.0                                                                                                             \n",
      " orgContr           | 999990849.0|249360.0 998732274.0|1275102.0                                                                               \n",
      " coordinatorCountry | NL                                                                                                                       \n",
      " coordinatorOrg     | 9.98732274E8                                                                                                             \n",
      " euroSciVocCode     | [657, 225]                                                                                                               \n",
      " publicationID      | [82DDA554B3C5AC05784C188727C5BF5C43EEC64A, 7A1CC89A6F17F93E1BF5046DE3D415A121696E61, 418603D7E067F89AF41A26E336E3E392... \n",
      " patentID           | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "projects = spark.read.parquet(path2cordis + \"projects.parquet\")\n",
    "print('Number of projects:', projects.count())\n",
    "\n",
    "cancer_codes = [1475, 1471, 86801484, 1469, 30053, 719, 1467, 49564263, 78799773, 1473, 30063, 1801, 30064, 1465, 225]\n",
    "\n",
    "#Keep only projects which contain non-null euroSciVocCode field\n",
    "cancer_projects = projects.filter(F.col(\"euroSciVocCode\").isNotNull())\n",
    "\n",
    "EVOC_match_UDF = F.udf(lambda x: len([el for el in x if el in cancer_codes]), IntegerType())\n",
    "\n",
    "cancer_projects = cancer_projects.withColumn(\"isCancer\", EVOC_match_UDF(F.col(\"euroSciVocCode\")))\n",
    "cancer_projects = cancer_projects.filter(cancer_projects.isCancer > 0).drop(\"isCancer\")\n",
    "\n",
    "print('Number of cancer projects according to EuroSciVoc:', cancer_projects.count())\n",
    "cancer_projects.printSchema()\n",
    "cancer_projects.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "caefdb0f-4151-4bf3-8e6a-b1ff15a63a1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cordis_EVOC_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_evoc_Cancer.parquet\"\n",
    "\n",
    "cancer_projects.repartition(40).write.parquet(cordis_EVOC_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c70163b-80be-40c4-8146-035630beefbb",
   "metadata": {},
   "source": [
    "#### 1.1.3.2. Filtering according to Keywords\n",
    "\n",
    "Additionally, we have a list of 123 keywords provided by HCERES that can be used for the selection of patents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3c6aebb7-6abf-4b03-8c95-4d21a05796f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of keywords: 123\n"
     ]
    }
   ],
   "source": [
    "with open(\"/export/clusterdata/jarenas/Datasets/patstat/2023_Spring/Cancer_keywords_HCERES_patents.txt\", \"r\") as fin:\n",
    "    Cancer_kwds = [el.strip() for el in fin.readlines()]\n",
    "    Cancer_kwds = [el for el in Cancer_kwds if len(el)]\n",
    "print('Number of keywords:', len(Cancer_kwds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2893763d-da46-44b1-93b5-922088bb4536",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 219:====================================================>  (42 + 2) / 44]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects with cancer keywords: 5121\n",
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 200620                                                                                                                   \n",
      " acronym            | CANCERDIP                                                                                                                \n",
      " status             | CLO                                                                                                                      \n",
      " title              | The use of Methylated DNA Immunoprecipitation MeDIP in cancer for better clinical management                             \n",
      " startDate          | 2008-01-01 00:00:00                                                                                                      \n",
      " endDate            | 2011-06-30 00:00:00                                                                                                      \n",
      " totalCost          | 3922042.97                                                                                                               \n",
      " ecMaxContribution  | 2999994.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-HEALTH-2007-A                                                                                                        \n",
      " fundingScheme      | CP-FP                                                                                                                    \n",
      " nature             | null                                                                                                                     \n",
      " objective          | Aberrant DNA methylation is the most common molecular lesion of the cancer cell. Neither gene mutation nor cytogeneti... \n",
      " contentUpdateDate  | 2019-07-16 09:49:00                                                                                                      \n",
      " rcn                | 88147                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | HEALTH-2007-2.4.1-3                                                                                                      \n",
      " topic_title        | Genomic instability and genomic alterations in pre-cancerous lesions and/or cancer                                       \n",
      " countryContr       | BE|915498.0 DE|299999.0 ES|825001.0 IT|299998.0 NL|659498.0                                                              \n",
      " orgContr           | 999547850.0|285657.69 999992110.0|659498.0 999755139.0|265499.0 999986290.0|649999.0 999848356.0|299998.0 999990267.0... \n",
      " coordinatorCountry | ES                                                                                                                       \n",
      " coordinatorOrg     | 9.99600618E8                                                                                                             \n",
      " euroSciVocCode     | [1015, 1017, 225, 1025, 44114657]                                                                                        \n",
      " publicationID      | [B61F89E82970C5F6EB084699CAA1D528C01FF9C6, 7221DC3EAE837A94075854E425AA0E4156E9C976, E97FC7F0213371ED4C52623199D50476... \n",
      " patentID           | null                                                                                                                     \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 220566                                                                                                                   \n",
      " acronym            | BCL6-C-JUN-CAX                                                                                                           \n",
      " status             | CLO                                                                                                                      \n",
      " title              | cooperation between BCL6 and c-Jun in lymphoma development and progression                                               \n",
      " startDate          | 2008-12-01 00:00:00                                                                                                      \n",
      " endDate            | 2010-11-30 00:00:00                                                                                                      \n",
      " totalCost          | 178307.05                                                                                                                \n",
      " ecMaxContribution  | 178307.05                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-PEOPLE-2007-2-1-IEF                                                                                                  \n",
      " fundingScheme      | MC-IEF                                                                                                                   \n",
      " nature             | null                                                                                                                     \n",
      " objective          | Sequential activation of different oncogenes drive the step-wise progression of precancerous cells to highly malignan... \n",
      " contentUpdateDate  | 2019-07-16 21:21:14                                                                                                      \n",
      " rcn                | 88683                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | PEOPLE-2007-2-1.IEF                                                                                                      \n",
      " topic_title        | Marie Curie Action: \"Intra-European Fellowships for Career Development\"                                                  \n",
      " countryContr       | UK|178307.05                                                                                                             \n",
      " orgContr           | 999494888.0|178307.05                                                                                                    \n",
      " coordinatorCountry | UK                                                                                                                       \n",
      " coordinatorOrg     | 9.99494888E8                                                                                                             \n",
      " euroSciVocCode     | [315, 145, 309]                                                                                                          \n",
      " publicationID      | null                                                                                                                     \n",
      " patentID           | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "def count_kwds(x):\n",
    "    try:\n",
    "        reps = [x.lower().count(k) for k in Cancer_kwds]\n",
    "        return sum(reps)\n",
    "    except:\n",
    "        return 0\n",
    "\n",
    "count_kwds_UDF = F.udf(count_kwds, IntegerType())\n",
    "\n",
    "cancer_projects = projects.withColumn('raw_text', \n",
    "                    F.concat(F.col('title'),F.lit(' '), F.col('objective')))\n",
    "\n",
    "cancer_projects = cancer_projects.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "cancer_projects = cancer_projects.filter(cancer_projects.Kwd_count > 0).cache().drop(\"raw_text\").drop(\"Kwd_count\")\n",
    "\n",
    "print('Number of projects with cancer keywords:', cancer_projects.count())\n",
    "cancer_projects.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "0615a25d-0857-475b-acfb-03362a26162a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cordis_Kwds_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_Kwds_Cancer.parquet\"\n",
    "\n",
    "cancer_projects.repartition(40).write.parquet(cordis_Kwds_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6b54731-4723-4b93-a805-237078db01b2",
   "metadata": {},
   "source": [
    "#### 1.1.3.3. Filtering according to acknowledgments to Cancer publications\n",
    "\n",
    "A last strategy is to select projects that include among their results some of the papers which have been selected by HCERES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "86c22ec4-bcef-4f4d-a5f7-ac2b6cff17a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Cancer publications in S2: 2436965\n",
      "-RECORD 0---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 19161039                                                                                                                 \n",
      " doi           | 10.1002/(SICI)1098-2264(200002)27:2<162::AID-GCC7>3.0.CO;2-9                                                             \n",
      " year          | 2000                                                                                                                     \n",
      " title         | Molecular genetic alterations in carcinoma ex‐pleomorphic adenoma: A putative progression model?                         \n",
      " paperAbstract | To determine the genetic changes associated with the development of carcinoma ex‐pleomorphic adenoma (Ca Ex‐PA), we a... \n",
      "-RECORD 1---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 46428546                                                                                                                 \n",
      " doi           | 10.1016/S0277-9536(99)00256-7                                                                                            \n",
      " year          | 2000                                                                                                                     \n",
      " title         | Rethinking patient satisfaction: patient experiences of an open access flexible sigmoidoscopy service.                   \n",
      " paperAbstract | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n",
      "-RECORD 0-----------------------------------------------------------\n",
      " doi | 10.1002/(SICI)1098-2264(200002)27:2<162::AID-GCC7>3.0.CO;2-9 \n",
      "-RECORD 1-----------------------------------------------------------\n",
      " doi | 10.1016/S0277-9536(99)00256-7                                \n",
      "-RECORD 2-----------------------------------------------------------\n",
      " doi | 10.1006/EXCR.1999.4736                                       \n",
      "-RECORD 3-----------------------------------------------------------\n",
      " doi | 10.1128/JVI.74.2.899-913.2000                                \n",
      "-RECORD 4-----------------------------------------------------------\n",
      " doi | 10.1016/S1383-5718(99)00174-6                                \n",
      "only showing top 5 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 89:=================================================>      (35 + 5) / 40]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Cancer publications with doi: 2310347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_S2 = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/S2_Cancer.parquet')\n",
    "\n",
    "print('Number of Cancer publications in S2:', cancer_S2.count())\n",
    "cancer_S2.show(n=2, truncate=120, vertical=True)\n",
    "\n",
    "cancer_dois = cancer_S2.filter(F.col(\"doi\").isNotNull()).select(\"doi\")\n",
    "cancer_dois.show(n=5, truncate=120, vertical=True)\n",
    "print('Number of Cancer publications with doi:', cancer_dois.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "90bec545-bf47-4bc3-a490-fc5fb24ddb0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications with doi: 626316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications in the cancer dataset: 84373\n",
      "root\n",
      " |-- doi: string (nullable = true)\n",
      " |-- projectID: long (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- authors: string (nullable = true)\n",
      " |-- isPublishedAs: string (nullable = true)\n",
      " |-- repositoryUrl: string (nullable = true)\n",
      " |-- journalTitle: string (nullable = true)\n",
      " |-- publisher: string (nullable = true)\n",
      " |-- journalNumber: string (nullable = true)\n",
      " |-- publishedPages: string (nullable = true)\n",
      " |-- id: string (nullable = true)\n",
      " |-- frameworkProgramme: string (nullable = true)\n",
      " |-- publishedYear: double (nullable = true)\n",
      " |-- issn: string (nullable = true)\n",
      " |-- isbn: string (nullable = true)\n",
      " |-- projectAcronym: string (nullable = true)\n",
      " |-- collection: string (nullable = true)\n",
      " |-- contentUpdateDate: timestamp (nullable = true)\n",
      " |-- rcn: double (nullable = true)\n",
      " |-- SSID: long (nullable = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 136:=====================================================> (34 + 1) / 35]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " doi                | 10.1002/adhm.201900665                                                                                                   \n",
      " projectID          | 319130                                                                                                                   \n",
      " title              | Supramolecular Toxin Complexes for Targeted Pharmacological Modulation of Polymorphonuclear Leukocyte Functions          \n",
      " authors            | Astrid Johanna Heck , Theresa Ostertag , Leonie Schnell , Stephan Fischer , Bikram Keshari Agrawalla , Pia Winterwerb... \n",
      " isPublishedAs      | PEER_REVIEWED_ARTICLE                                                                                                    \n",
      " repositoryUrl      | https://onlinelibrary.wiley.com/doi/abs/10.1002/adhm.201900665                                                           \n",
      " journalTitle       | Advanced Healthcare Materials                                                                                            \n",
      " publisher          | John Wiley and Sons Ltd                                                                                                  \n",
      " journalNumber      | null                                                                                                                     \n",
      " publishedPages     | null                                                                                                                     \n",
      " id                 | D09CDDC9CB2CAE4585723409F648E2A7431DC19D                                                                                 \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " publishedYear      | null                                                                                                                     \n",
      " issn               | null                                                                                                                     \n",
      " isbn               | null                                                                                                                     \n",
      " projectAcronym     | null                                                                                                                     \n",
      " collection         | null                                                                                                                     \n",
      " contentUpdateDate  | null                                                                                                                     \n",
      " rcn                | null                                                                                                                     \n",
      " SSID               | 197542916                                                                                                                \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " doi                | 10.1002/chem.201502809                                                                                                   \n",
      " projectID          | 309782                                                                                                                   \n",
      " title              | Light-Controlled Histone Deacetylase (HDAC) Inhibitors: Towards Photopharmacological Chemotherapy                        \n",
      " authors            | Wiktor Szymanski , Maria E. Ourailidou , Willem A. Velema , Frank J. Dekker , Ben L. Feringa                             \n",
      " isPublishedAs      | PEER_REVIEWED_ARTICLE                                                                                                    \n",
      " repositoryUrl      | http://doi.wiley.com/10.1002/chem.201502809                                                                              \n",
      " journalTitle       | Chemistry - A European Journal                                                                                           \n",
      " publisher          | John Wiley & Sons Ltd.                                                                                                   \n",
      " journalNumber      | null                                                                                                                     \n",
      " publishedPages     | null                                                                                                                     \n",
      " id                 | 9192F8F30EE30ABBEFB4A5F829DB17A234FC1EEC                                                                                 \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " publishedYear      | null                                                                                                                     \n",
      " issn               | null                                                                                                                     \n",
      " isbn               | null                                                                                                                     \n",
      " projectAcronym     | null                                                                                                                     \n",
      " collection         | null                                                                                                                     \n",
      " contentUpdateDate  | null                                                                                                                     \n",
      " rcn                | null                                                                                                                     \n",
      " SSID               | 205616475                                                                                                                \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cordis_publis = spark.read.parquet(path2cordis + \"publications.parquet\")\n",
    "\n",
    "#We filter those without doi\n",
    "cordis_publis = cordis_publis.filter(F.col(\"doi\").isNotNull())\n",
    "print('Number of publications with doi:', cordis_publis.count())\n",
    "\n",
    "#We retain only the citations related to cancer\n",
    "cancer_publis = cordis_publis.join(cancer_dois, on=\"doi\", how=\"inner\").drop(cancer_dois.doi)\n",
    "print('Number of publications in the cancer dataset:', cancer_publis.count())\n",
    "\n",
    "cancer_publis.printSchema()\n",
    "cancer_publis.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1b3be19b-75d5-4a22-b1ab-586b8170ef2a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 145:==================================================>    (40 + 4) / 44]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of selected projects: 4097\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "projects_publis = cancer_publis.select(\"projectID\").distinct()\n",
    "print(\"Number of selected projects:\", projects_publis.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "70f3f76f-a29d-4afc-b8d3-731c1eac021e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects ackwnoledged by cancer papers: 4095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 175:===========================================>           (35 + 9) / 44]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 223367                                                                                                                   \n",
      " acronym            | MULTIMOD                                                                                                                 \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Multi-layer network modules to identify markers for personalized medication in complex diseases                          \n",
      " startDate          | 2008-11-01 00:00:00                                                                                                      \n",
      " endDate            | 2012-10-31 00:00:00                                                                                                      \n",
      " totalCost          | 3644610.4                                                                                                                \n",
      " ecMaxContribution  | 2525888.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-HEALTH-2007-B                                                                                                        \n",
      " fundingScheme      | CP-FP                                                                                                                    \n",
      " nature             | null                                                                                                                     \n",
      " objective          | The symptoms of complex disease like allergy, obesity and cancer depend on the products of multiple interacting genes... \n",
      " contentUpdateDate  | 2019-07-16 19:23:18                                                                                                      \n",
      " rcn                | 90359                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | HEALTH-2007-2.1.2-5                                                                                                      \n",
      " topic_title        | Multidisciplinary fundamental genomics and molecular biology approaches to study basic biological processes relevant ... \n",
      " countryContr       | DE|166800.0 NO|734919.0 SE|1270529.0 UK|353640.0 US|0.0                                                                  \n",
      " orgContr           | 888897696.0|492991.2 999993468.0|353640.0 991104000.0|734919.0 998465136.0|0.0 998464845.0|166800.0 999981925.0|777537.8 \n",
      " coordinatorCountry | SE                                                                                                                       \n",
      " coordinatorOrg     | 8.88897696E8                                                                                                             \n",
      " euroSciVocCode     | [653, 1613, 1019, 1021, 38005658]                                                                                        \n",
      " publicationID      | [CEEB8921514B760BD721502290678DBFB0F6D3C9, 13ACF1A668641B540FA64FB17B783BA93D601645, 235849DF06B07FE43E91D919E5270376... \n",
      " patentID           | null                                                                                                                     \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 247091                                                                                                                   \n",
      " acronym            | MIRACLE                                                                                                                  \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Microscopic Image Processing, Analysis, Classification and Modelling Environment                                         \n",
      " startDate          | 2010-05-01 00:00:00                                                                                                      \n",
      " endDate            | 2014-02-28 00:00:00                                                                                                      \n",
      " totalCost          | 212000.0                                                                                                                 \n",
      " ecMaxContribution  | 189000.0                                                                                                                 \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-PEOPLE-2009-IRSES                                                                                                    \n",
      " fundingScheme      | MC-IRSES                                                                                                                 \n",
      " nature             | null                                                                                                                     \n",
      " objective          | The main objective of this exchange scheme is to establish a long-term research co-operation at an international leve... \n",
      " contentUpdateDate  | 2023-03-10 21:20:28                                                                                                      \n",
      " rcn                | 97183                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | FP7-PEOPLE-2009-IRSES                                                                                                    \n",
      " topic_title        | Marie Curie Action \"International Research Staff Exchange Scheme\"                                                        \n",
      " countryContr       | EL|45000.0 FR|32400.0 TR|70200.0 UK|41400.0                                                                              \n",
      " orgContr           | 998802502.0|45000.0 999923628.0|70200.0 998737124.0|32400.0 999976784.0|41400.0                                          \n",
      " coordinatorCountry | TR                                                                                                                       \n",
      " coordinatorOrg     | 9.99923628E8                                                                                                             \n",
      " euroSciVocCode     | [225, 50814433, 41594476]                                                                                                \n",
      " publicationID      | [733C82679485D86B87E8A54EE9C71935F37B3507, E38696FF2682BBF39062142749FEAEAE1A9E742D, C9830C875CB992023609DF9B78B8C61C... \n",
      " patentID           | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_projects = projects.join(projects_publis, on=\"projectID\", how=\"inner\").drop(projects_publis.projectID)\n",
    "\n",
    "print('Number of projects ackwnoledged by cancer papers:', cancer_projects.count())\n",
    "cancer_projects.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4f7ff0c1-1643-46c0-a552-09ab2d99cdaa",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cordis_ACK_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_ACK_Cancer.parquet\"\n",
    "\n",
    "cancer_projects.repartition(40).write.parquet(cordis_ACK_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efa95e28-7bdc-4ad7-bfe3-634a2d2d4880",
   "metadata": {},
   "source": [
    "#### 1.1.3.4. Calculating the intersection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "48d03f27-ce4c-4ccb-9726-496b2344bc33",
   "metadata": {},
   "outputs": [],
   "source": [
    "cancer_evoc = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_evoc_Cancer.parquet')\n",
    "cancer_Kwds = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_Kwds_Cancer.parquet')\n",
    "cancer_ACK = spark.read.parquet('file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_ACK_Cancer.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "edf2788a-7787-4709-86fe-38d89f338293",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of intersection: 1308\n",
      "root\n",
      " |-- projectID: long (nullable = true)\n",
      " |-- acronym: string (nullable = true)\n",
      " |-- status: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- startDate: timestamp (nullable = true)\n",
      " |-- endDate: timestamp (nullable = true)\n",
      " |-- totalCost: string (nullable = true)\n",
      " |-- ecMaxContribution: string (nullable = true)\n",
      " |-- ecSignatureDate: timestamp (nullable = true)\n",
      " |-- frameworkProgramme: string (nullable = true)\n",
      " |-- masterCall: string (nullable = true)\n",
      " |-- subCall: string (nullable = true)\n",
      " |-- fundingScheme: string (nullable = true)\n",
      " |-- nature: string (nullable = true)\n",
      " |-- objective: string (nullable = true)\n",
      " |-- contentUpdateDate: timestamp (nullable = true)\n",
      " |-- rcn: long (nullable = true)\n",
      " |-- grantDoi: string (nullable = true)\n",
      " |-- topic: string (nullable = true)\n",
      " |-- topic_title: string (nullable = true)\n",
      " |-- countryContr: string (nullable = true)\n",
      " |-- orgContr: string (nullable = true)\n",
      " |-- coordinatorCountry: string (nullable = true)\n",
      " |-- coordinatorOrg: double (nullable = true)\n",
      " |-- euroSciVocCode: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- publicationID: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- patentID: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      "\n",
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 261506                                                                                                                   \n",
      " acronym            | AIPGENE                                                                                                                  \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Augmenting PBGD expression in the liver as a Novel Gene therapy for Acute Intermittent Porphyria                         \n",
      " startDate          | 2011-01-01 00:00:00                                                                                                      \n",
      " endDate            | 2014-10-31 00:00:00                                                                                                      \n",
      " totalCost          | 4347949.14                                                                                                               \n",
      " ecMaxContribution  | 3308154.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-HEALTH-2010-single-stage                                                                                             \n",
      " fundingScheme      | CP-FP                                                                                                                    \n",
      " nature             | null                                                                                                                     \n",
      " objective          | Acute intermittent porphyria (AIP) is a rare genetic disease in which mutations in the porphobilinogen deaminase (PBG... \n",
      " contentUpdateDate  | 2019-08-02 14:31:10                                                                                                      \n",
      " rcn                | 98834                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | HEALTH.2010.2.4.4-1                                                                                                      \n",
      " topic_title        | Clinical development of substances with a clear potential as orphan drugs. FP7-HEALTH-2010-single-stage                  \n",
      " countryContr       | DE|321100.0 ES|1825042.0 NL|1092012.0 SE|70000.0                                                                         \n",
      " orgContr           | 984947216.0|448000.0 999485382.0|822842.0 999990073.0|321100.0 999481987.0|45400.0 999641746.0|508800.0 999680061.0|1... \n",
      " coordinatorCountry | ES                                                                                                                       \n",
      " coordinatorOrg     | 9.99485382E8                                                                                                             \n",
      " euroSciVocCode     | [1467, 659, 325, 237, 153]                                                                                               \n",
      " publicationID      | [FB7C968AC4CB16F40802A3C47194F00474B0312A, 79902EB3CEBCE28CB16EA49F4F5A62D0A6D51608, 5EACC50EEB48DF2D52A5133695D06378... \n",
      " patentID           | [40130866, 40130866]                                                                                                     \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 335326                                                                                                                   \n",
      " acronym            | MacTherVac                                                                                                               \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Modulation of a novel population of immune suppressive tumoural macrophages and the therapeutic vaccination of cancer    \n",
      " startDate          | 2013-09-01 00:00:00                                                                                                      \n",
      " endDate            | 2018-08-31 00:00:00                                                                                                      \n",
      " totalCost          | 1499994.0                                                                                                                \n",
      " ecMaxContribution  | 1499994.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | ERC-2013-StG                                                                                                             \n",
      " fundingScheme      | ERC-SG                                                                                                                   \n",
      " nature             | null                                                                                                                     \n",
      " objective          | The therapeutic vaccination against breast cancer, and more widely all solid tumours, has largely been ineffective in... \n",
      " contentUpdateDate  | 2016-03-31 17:04:20                                                                                                      \n",
      " rcn                | 109963                                                                                                                   \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | ERC-SG-LS7                                                                                                               \n",
      " topic_title        | Applied life sciences, biotechnology and bioengineering: agricultural, animal, fishery, forestry/food sciences; biote... \n",
      " countryContr       | UK|1499994.0                                                                                                             \n",
      " orgContr           | 999981052.0|1499994.0                                                                                                    \n",
      " coordinatorCountry | UK                                                                                                                       \n",
      " coordinatorOrg     | 9.99981052E8                                                                                                             \n",
      " euroSciVocCode     | [49564263, 30816592, 1471, 1439, 643]                                                                                    \n",
      " publicationID      | [B0F3B3EB32894675EABC904BAAE49649C4BC58A0, E1A68D2A46C07195E8CC63E3B604B283C675EB0E, 9457108B223B50F51D96512E7990A32B... \n",
      " patentID           | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_aux = cancer_Kwds.select(\"projectID\")\n",
    "cancer_intersection = cancer_evoc.join(df_aux, on = \"projectID\", how = \"inner\").drop(df_aux.projectID)\n",
    "df_aux = cancer_ACK.select(\"projectID\")\n",
    "cancer_intersection = cancer_intersection.join(df_aux, on = \"projectID\", how = \"inner\").drop(df_aux.projectID)\n",
    "\n",
    "print('Size of intersection:', cancer_intersection.count())\n",
    "cancer_intersection.printSchema()\n",
    "cancer_intersection.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ae772e84-b263-4dd6-aad8-d83cc37f859b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_intersection_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_intersection_Cancer.parquet\"\n",
    "\n",
    "cancer_intersection.repartition(40).write.parquet(cancer_intersection_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f11ca75-d88e-4ea6-8908-1a4f1d44dd0a",
   "metadata": {},
   "source": [
    "#### 1.1.3.5. Calculating the union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "28810f6b-7cc2-457f-8ae5-9a70052f81c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of union: 7722\n",
      "root\n",
      " |-- projectID: long (nullable = true)\n",
      " |-- acronym: string (nullable = true)\n",
      " |-- status: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- startDate: timestamp (nullable = true)\n",
      " |-- endDate: timestamp (nullable = true)\n",
      " |-- totalCost: string (nullable = true)\n",
      " |-- ecMaxContribution: string (nullable = true)\n",
      " |-- ecSignatureDate: timestamp (nullable = true)\n",
      " |-- frameworkProgramme: string (nullable = true)\n",
      " |-- masterCall: string (nullable = true)\n",
      " |-- subCall: string (nullable = true)\n",
      " |-- fundingScheme: string (nullable = true)\n",
      " |-- nature: string (nullable = true)\n",
      " |-- objective: string (nullable = true)\n",
      " |-- contentUpdateDate: timestamp (nullable = true)\n",
      " |-- rcn: long (nullable = true)\n",
      " |-- grantDoi: string (nullable = true)\n",
      " |-- topic: string (nullable = true)\n",
      " |-- topic_title: string (nullable = true)\n",
      " |-- countryContr: string (nullable = true)\n",
      " |-- orgContr: string (nullable = true)\n",
      " |-- coordinatorCountry: string (nullable = true)\n",
      " |-- coordinatorOrg: double (nullable = true)\n",
      " |-- euroSciVocCode: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- publicationID: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- patentID: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 235:=====================================>               (85 + 35) / 120]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 115153                                                                                                                   \n",
      " acronym            | RAPP-ID                                                                                                                  \n",
      " status             | ONG                                                                                                                      \n",
      " title              | Development of RApid Point-of-Care test Platforms for Infectious Diseases                                                \n",
      " startDate          | 2011-04-01 00:00:00                                                                                                      \n",
      " endDate            | 2016-09-30 00:00:00                                                                                                      \n",
      " totalCost          | 1.59273E7                                                                                                                \n",
      " ecMaxContribution  | 6828438.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | IMI-JU-02-2009                                                                                                           \n",
      " fundingScheme      | JTI-CP-IMI                                                                                                               \n",
      " nature             | null                                                                                                                     \n",
      " objective          | RAPP-ID (“Development of RApid Point-of-Care test Platforms for Infectious Diseases”) will develop a Point-of-Care Te... \n",
      " contentUpdateDate  | 2022-03-25 22:16:24                                                                                                      \n",
      " rcn                | 203690                                                                                                                   \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | IMI-JU-02-2009-04                                                                                                        \n",
      " topic_title        | Infectious Diseases – Diagnostic Tools                                                                                   \n",
      " countryContr       | BE|2327025.0 CH|60500.0 DE|944300.0 FI|174455.0 FR|0.0 IT|0.0 NL|364003.0 SE|2393622.0 UK|564533.0 US|0.0                \n",
      " orgContr           | 999902870.0|1272684.0 999977172.0|489154.0 997769840.0|491845.0 999990946.0|1327277.0 999939633.0|0.0 999988618.0|0.0... \n",
      " coordinatorCountry | BE                                                                                                                       \n",
      " coordinatorOrg     | 9.98398206E8                                                                                                             \n",
      " euroSciVocCode     | [1035, 121, 1031, 709, 1437]                                                                                             \n",
      " publicationID      | [AF90E5DC4ADECE6436803DED3570DFBF8A15ADCF, DB967A7DA5C14C31A888F36CF7E2609490A814F8, FA995971C057DDF015FB54BE4BF2B770... \n",
      " patentID           | null                                                                                                                     \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 116030                                                                                                                   \n",
      " acronym            | TransQST                                                                                                                 \n",
      " status             | SIGNED                                                                                                                   \n",
      " title              | Translational quantitative systems toxicology to improve the understanding of the safety of medicines - Sofia: 116030    \n",
      " startDate          | 2017-01-01 00:00:00                                                                                                      \n",
      " endDate            | 2022-08-31 00:00:00                                                                                                      \n",
      " totalCost          | 1.8132874E7                                                                                                              \n",
      " ecMaxContribution  | 8000000.0                                                                                                                \n",
      " ecSignatureDate    | 2016-12-22 00:00:00                                                                                                      \n",
      " frameworkProgramme | H2020                                                                                                                    \n",
      " masterCall         | H2020-JTI-IMI2-2015-06-two-stage                                                                                         \n",
      " subCall            | H2020-JTI-IMI2-2015-06-two-stage                                                                                         \n",
      " fundingScheme      | RIA                                                                                                                      \n",
      " nature             | crisisPreparedness                                                                                                       \n",
      " objective          | TransQST will develop a Quantitative Systems Toxicology (QST) approach, employing pre-existing data where possible, i... \n",
      " contentUpdateDate  | 2022-12-31 18:28:15                                                                                                      \n",
      " rcn                | 209467                                                                                                                   \n",
      " grantDoi           | 10.3030/116030                                                                                                           \n",
      " topic              | IMI2-2015-06-01                                                                                                          \n",
      " topic_title        | Development of Quantitative System Toxicology (QST) approaches to improve the understanding of the safety of new medi... \n",
      " countryContr       | AT|350250.0 BE|0.0 DE|1845563.75 ES|1254193.75 FI|0.0 FR|0.0 NL|2578478.7499999995 SE|0.0 UK|1971513.75                  \n",
      " orgContr           | 951246021|0.0 999974553|1239324.43 999897632|132435.0 999939633|0.0 890911342|0.0 999988230|1093088.75 999522533|4846... \n",
      " coordinatorCountry | UK                                                                                                                       \n",
      " coordinatorOrg     | 9.99980567E8                                                                                                             \n",
      " euroSciVocCode     | [79909316, 645, 38221291, 651, 147]                                                                                      \n",
      " publicationID      | [116030_1422125_PUBLI, 116030_352930_PUBLI, 116030_1332381_PUBLI, 116030_849512_PUBLI, 116030_1040516_PUBLI, 116030_1... \n",
      " patentID           | null                                                                                                                     \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_all = cancer_evoc.union(cancer_Kwds).union(cancer_ACK).dropDuplicates([\"projectID\"])\n",
    "\n",
    "print('Size of union:', cancer_all.count())\n",
    "cancer_all.printSchema()\n",
    "cancer_all.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "282f5493-5e80-476f-9b57-20ae323a48d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- projectID: long (nullable = true)\n",
      " |-- acronym: string (nullable = true)\n",
      " |-- status: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- startDate: timestamp (nullable = true)\n",
      " |-- endDate: timestamp (nullable = true)\n",
      " |-- totalCost: string (nullable = true)\n",
      " |-- ecMaxContribution: string (nullable = true)\n",
      " |-- ecSignatureDate: timestamp (nullable = true)\n",
      " |-- frameworkProgramme: string (nullable = true)\n",
      " |-- masterCall: string (nullable = true)\n",
      " |-- subCall: string (nullable = true)\n",
      " |-- fundingScheme: string (nullable = true)\n",
      " |-- nature: string (nullable = true)\n",
      " |-- objective: string (nullable = true)\n",
      " |-- contentUpdateDate: timestamp (nullable = true)\n",
      " |-- rcn: long (nullable = true)\n",
      " |-- grantDoi: string (nullable = true)\n",
      " |-- topic: string (nullable = true)\n",
      " |-- topic_title: string (nullable = true)\n",
      " |-- countryContr: string (nullable = true)\n",
      " |-- orgContr: string (nullable = true)\n",
      " |-- coordinatorCountry: string (nullable = true)\n",
      " |-- coordinatorOrg: double (nullable = true)\n",
      " |-- euroSciVocCode: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- publicationID: array (nullable = true)\n",
      " |    |-- element: string (containsNull = true)\n",
      " |-- patentID: array (nullable = true)\n",
      " |    |-- element: long (containsNull = true)\n",
      " |-- raw_text: string (nullable = true)\n",
      " |-- Kwd_count: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "40f92cf6-3368-4060-846d-bc7f8e022267",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "cancer_all_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/cancer/cordis_all_Cancer.parquet\"\n",
    "\n",
    "cancer_all.repartition(40).write.parquet(cancer_all_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0297009d-c8a6-468d-a6eb-e94c0624a3c5",
   "metadata": {},
   "source": [
    "# 2. Artificial Intelligence\n",
    "\n",
    "In all cases, matches in this domain are started with a series of AI specific keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1b0f9d62-5cbf-478c-96e8-b8c9544d5216",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of keywords: 266\n"
     ]
    }
   ],
   "source": [
    "with open(\"/export/data_ml4ds/IntelComp/Datasets/semanticscholar/20230418/IA_keywords_SEAD_REV_JAG_JCS.txt\", \"r\") as fin:\n",
    "    AI_kwds = [el.strip() for el in fin.readlines()]\n",
    "    AI_kwds = [el for el in AI_kwds if len(el)]\n",
    "print('Number of keywords:', len(AI_kwds))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "516e2d6d-c612-4f67-a338-7b84a1905c0d",
   "metadata": {},
   "source": [
    "## 2.1. Domain Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16824f94-6be4-4a34-aa2c-42b2a7772cfb",
   "metadata": {},
   "source": [
    "### 2.1.1 Publications"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6a72ede-06e8-4d8c-a509-a7da9907e722",
   "metadata": {},
   "source": [
    "#### 2.1.1.1. Semantic Scholar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d2bf168c-b49f-4e0e-9ef9-9ce7d28a791d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/09/06 22:49:58 WARN CacheManager: Asked to cache already cached data.        \n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in AI dataset (Semantic Scholar): 2128190\n",
      "-RECORD 0---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 3091                                                                                                                     \n",
      " pmid          | null                                                                                                                     \n",
      " doi           | 10.1145/505282.505283                                                                                                    \n",
      " year          | 2001                                                                                                                     \n",
      " title         | Machine learning in automated text categorization                                                                        \n",
      " paperAbstract | The automated categorization (or classification) of texts into predefined categories has witnessed a booming interest... \n",
      " Kwd_count     | 6                                                                                                                        \n",
      "-RECORD 1---------------------------------------------------------------------------------------------------------------------------------\n",
      " id            | 11567                                                                                                                    \n",
      " pmid          | null                                                                                                                     \n",
      " doi           | 10.1109/ICASSP.2011.5947457                                                                                              \n",
      " year          | 2011                                                                                                                     \n",
      " title         | Powerful extensions to CRFS for grapheme to phoneme conversion                                                           \n",
      " paperAbstract | Conditional Random Fields (CRFs) have proven to perform well on natural language processing tasks like name translite... \n",
      " Kwd_count     | 2                                                                                                                        \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Matching with Semantic Scholar publications\n",
    "\n",
    "papers_S2 = spark.read.parquet(path2S2 + \"papers.parquet\")\n",
    "col_to_keep = [\"id\", \"pmid\", \"doi\", \"year\", \"title\", \"paperAbstract\"]\n",
    "papers_S2 = papers_S2.select(*col_to_keep).cache()\n",
    "\n",
    "def count_kwds(x):\n",
    "    try:\n",
    "        reps = [x.lower().count(k) for k in AI_kwds]\n",
    "        return sum(reps)\n",
    "    except:\n",
    "        return 0\n",
    "\n",
    "count_kwds_UDF = F.udf(count_kwds, IntegerType())\n",
    "\n",
    "AI_papers_S2 = papers_S2.withColumn('raw_text', \n",
    "                    F.concat(F.col('title'),F.lit(' '), F.col('paperAbstract')))\n",
    "\n",
    "AI_papers_S2 = AI_papers_S2.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "AI_papers_S2 = AI_papers_S2.drop(\"raw_text\").filter(AI_papers_S2.Kwd_count > 0).cache()\n",
    "\n",
    "print('Number of documents in AI dataset (Semantic Scholar):', AI_papers_S2.count())\n",
    "AI_papers_S2.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8b9ff316-ed86-46e8-863d-9a767b79f9ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "AI_S2_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/S2_Kwds_AI.parquet\"\n",
    "AI_papers_S2.write.parquet(AI_S2_file)\n",
    "\n",
    "AI_S2_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/S2_Kwds3_AI.parquet\"\n",
    "AI_papers_S2.filter(AI_papers_S2.Kwd_count > 2).write.parquet(AI_S2_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "465cff30-95b0-49ce-901f-f7b3b1c66bcc",
   "metadata": {},
   "source": [
    "#### 2.1.1.2. OpenAIRE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "99d08d78-6ee0-434c-9f15-aefd85849d9b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/09/07 01:09:14 WARN SharedInMemoryCache: Evicting cached table partition metadata from memory due to size constraints (spark.sql.hive.filesourcePartitionFileCacheSize = 262144000 bytes). This may impact query planning performance.\n",
      "23/09/07 01:22:27 ERROR TaskSchedulerImpl: Lost executor 1 on node60.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:27 WARN TaskSetManager: Lost task 15.0 in stage 17.0 (TID 5415) (node60.cluster.tsc.uc3m.es executor 1): ExecutorLostFailure (executor 1 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:27 WARN TaskSetManager: Lost task 5.0 in stage 17.0 (TID 5405) (node60.cluster.tsc.uc3m.es executor 1): ExecutorLostFailure (executor 1 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:27 WARN TaskSetManager: Lost task 35.0 in stage 17.0 (TID 5435) (node60.cluster.tsc.uc3m.es executor 1): ExecutorLostFailure (executor 1 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:27 WARN TaskSetManager: Lost task 25.0 in stage 17.0 (TID 5425) (node60.cluster.tsc.uc3m.es executor 1): ExecutorLostFailure (executor 1 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_6 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_393 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_542 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_268 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_796 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_142 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_93 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_502 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_541 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_51 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_259 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_344 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_32 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_200 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_41 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_796 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_461 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_550 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_147 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_651 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_339 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_542 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_193 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_70 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_126 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_248 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_336 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_126 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_651 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_801 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_449 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_312 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_147 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_712 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_506 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_216 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_216 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_32 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_796 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_166 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_253 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_229 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_664 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_45 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_195 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_466 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_494 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_32 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_358 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_133 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_48 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_172 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_393 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_466 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_503 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_384 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_103 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_464 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_93 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_752 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_265 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_220 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_45 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_358 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_560 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_17 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_172 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_300 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_664 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_195 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_320 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_111 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_105 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_342 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_113 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_687 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_730 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_752 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_339 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_798 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_265 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_460 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_111 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_6 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_384 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_25 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_76 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_346 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_608 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_333 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_219 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_72 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_142 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_177 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_297 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_248 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_219 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_300 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_610 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_76 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_228 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_268 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_88 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_284 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_507 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_597 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_99 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_645 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_636 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_344 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_140 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_648 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_229 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_185 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_486 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_697 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_268 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_746 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_346 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_72 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_166 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_282 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_793 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_17 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_99 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_506 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_491 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_518 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_687 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_312 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_91 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_394 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_103 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_560 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_744 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_793 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_216 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_376 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_41 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_559 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_172 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_126 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_712 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_185 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_460 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_193 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_41 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_381 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_296 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_653 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_99 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_785 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_140 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_352 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_742 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_302 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_745 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_601 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_51 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_58 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_806 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_555 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_218 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_704 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_741 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_48 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_254 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_391 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_449 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_147 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_48 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_687 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_302 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_387 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_140 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_461 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_590 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_187 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_612 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_745 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_546 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_325 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_612 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_428 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_164 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_25 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_793 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_742 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_662 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_208 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_75 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_704 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_806 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_59 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_325 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_6 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_657 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_675 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_518 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_657 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_25 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_208 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_185 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_442 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_449 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_555 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_746 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_460 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_494 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_70 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_81 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_708 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_801 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_259 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_428 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_610 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_560 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_228 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_603 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_590 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_170 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_708 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_70 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_559 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_225 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_752 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_248 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_81 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_391 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_17 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_196 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_645 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_503 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_608 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_282 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_589 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_708 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_177 !\n",
      "23/09/07 01:22:27 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_300 !\n",
      "23/09/07 01:22:30 ERROR TaskSchedulerImpl: Lost executor 8 on node67.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:30 WARN TaskSetManager: Lost task 12.0 in stage 17.0 (TID 5412) (node67.cluster.tsc.uc3m.es executor 8): ExecutorLostFailure (executor 8 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:30 WARN TaskSetManager: Lost task 32.0 in stage 17.0 (TID 5432) (node67.cluster.tsc.uc3m.es executor 8): ExecutorLostFailure (executor 8 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:30 WARN TaskSetManager: Lost task 2.0 in stage 17.0 (TID 5402) (node67.cluster.tsc.uc3m.es executor 8): ExecutorLostFailure (executor 8 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:30 WARN TaskSetManager: Lost task 22.0 in stage 17.0 (TID 5422) (node67.cluster.tsc.uc3m.es executor 8): ExecutorLostFailure (executor 8 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_375 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_194 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_44 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_3 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_443 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_28 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_164 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_552 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_625 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_107 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_21 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_66 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_726 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_492 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_144 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_240 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_272 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_180 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_194 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_353 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_297 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_375 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_226 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_398 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_77 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_270 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_567 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_321 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_748 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_44 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_191 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_781 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_615 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_479 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_605 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_143 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_101 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_726 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_177 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_647 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_792 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_619 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_336 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_270 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_463 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_40 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_398 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_107 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_728 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_671 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_112 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_385 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_78 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_391 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_363 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_82 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_724 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_552 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_597 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_226 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_724 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_351 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_13 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_304 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_208 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_144 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_112 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_272 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_50 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_297 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_176 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_540 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_133 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_250 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_77 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_584 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_735 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_775 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_792 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_100 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_106 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_492 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_50 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_629 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_563 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_343 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_209 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_692 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_180 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_443 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_21 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_237 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_50 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_143 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_211 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_28 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_454 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_341 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_201 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_674 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_584 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_656 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_79 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_470 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_385 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_647 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_619 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_43 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_28 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_519 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_629 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_781 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_166 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_256 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_69 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_694 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_312 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_694 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_483 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_680 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_240 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_416 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_499 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_287 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_521 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_332 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_583 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_223 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_314 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_671 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_778 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_563 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_440 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_132 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_351 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_13 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_220 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_351 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_76 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_552 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_133 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_671 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_176 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_674 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_468 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_97 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_748 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_762 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_554 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_659 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_144 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_226 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_182 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_142 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_417 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_726 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_57 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_3 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_307 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_125 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_220 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_314 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_112 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_169 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_615 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_191 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_195 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_724 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_792 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_529 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_326 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_441 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_239 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_100 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_567 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_521 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_3 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_256 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_272 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_599 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_519 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_768 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_533 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_223 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_21 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_321 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_533 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_353 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_66 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_271 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_373 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_768 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_499 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_713 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_169 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_629 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_768 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_125 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_473 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_57 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_51 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_578 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_417 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_174 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_164 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_680 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_13 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_516 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_728 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_336 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_680 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_57 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_40 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_79 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_473 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_597 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_201 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_80 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_341 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_509 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_470 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_107 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_270 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_271 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_97 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_304 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_206 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_285 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_775 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_209 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_69 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_454 !\n",
      "23/09/07 01:22:30 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_146 !\n",
      "23/09/07 01:22:37 ERROR TaskSchedulerImpl: Lost executor 6 on node00.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:37 WARN TaskSetManager: Lost task 27.0 in stage 17.0 (TID 5427) (node00.cluster.tsc.uc3m.es executor 6): ExecutorLostFailure (executor 6 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:37 WARN TaskSetManager: Lost task 17.0 in stage 17.0 (TID 5417) (node00.cluster.tsc.uc3m.es executor 6): ExecutorLostFailure (executor 6 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:37 WARN TaskSetManager: Lost task 37.0 in stage 17.0 (TID 5437) (node00.cluster.tsc.uc3m.es executor 6): ExecutorLostFailure (executor 6 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:37 WARN TaskSetManager: Lost task 7.0 in stage 17.0 (TID 5407) (node00.cluster.tsc.uc3m.es executor 6): ExecutorLostFailure (executor 6 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_299 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_709 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_357 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_485 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_787 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_766 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_586 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_350 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_591 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_366 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_604 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_732 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_333 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_154 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_527 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_255 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_53 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_732 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_711 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_754 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_709 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_377 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_542 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_8 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_366 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_207 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_782 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_490 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_653 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_60 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_102 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_102 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_350 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_321 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_617 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_548 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_711 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_422 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_604 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_165 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_207 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_461 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_709 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_399 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_422 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_459 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_360 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_367 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_445 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_747 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_203 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_558 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_62 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_650 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_8 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_333 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_441 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_762 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_469 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_471 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_299 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_595 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_600 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_255 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_478 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_203 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_475 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_558 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_485 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_323 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_377 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_295 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_527 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_387 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_736 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_789 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_330 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_337 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_782 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_711 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_664 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_707 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_651 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_703 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_305 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_782 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_535 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_682 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_604 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_71 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_754 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_129 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_622 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_762 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_598 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_526 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_441 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_541 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_616 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_23 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_396 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_747 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_595 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_399 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_650 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_654 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_595 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_736 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_337 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_637 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_620 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_682 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_445 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_637 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_274 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_600 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_154 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_528 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_526 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_366 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_535 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_129 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_305 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_707 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_789 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_203 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_471 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_384 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_725 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_703 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_747 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_60 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_274 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_469 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_510 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_694 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_575 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_331 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_305 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_345 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_622 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_374 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_670 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_295 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_586 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_745 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_787 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_563 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_716 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_165 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_511 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_448 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_71 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_302 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_654 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_779 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_23 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_399 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_53 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_62 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_165 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_627 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_686 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_622 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_652 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_620 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_654 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_257 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_257 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_558 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_307 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_686 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_154 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_307 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_274 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_591 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_653 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_350 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_511 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_526 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_469 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_374 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_8 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_295 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_754 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_712 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_23 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_60 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_71 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_357 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_575 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_576 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_129 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_489 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_102 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_627 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_255 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_422 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_600 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_257 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_755 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_541 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_766 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_387 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_475 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_445 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_471 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_53 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_459 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_766 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_620 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_487 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_330 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_62 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_489 !\n",
      "23/09/07 01:22:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_207 !\n",
      "23/09/07 01:22:41 ERROR TaskSchedulerImpl: Lost executor 7 on node74.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:41 WARN TaskSetManager: Lost task 3.0 in stage 17.0 (TID 5403) (node74.cluster.tsc.uc3m.es executor 7): ExecutorLostFailure (executor 7 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:41 WARN TaskSetManager: Lost task 33.0 in stage 17.0 (TID 5433) (node74.cluster.tsc.uc3m.es executor 7): ExecutorLostFailure (executor 7 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:41 WARN TaskSetManager: Lost task 23.0 in stage 17.0 (TID 5423) (node74.cluster.tsc.uc3m.es executor 7): ExecutorLostFailure (executor 7 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:41 WARN TaskSetManager: Lost task 13.0 in stage 17.0 (TID 5413) (node74.cluster.tsc.uc3m.es executor 7): ExecutorLostFailure (executor 7 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_596 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_18 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_652 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_667 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_304 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_47 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_472 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_54 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_210 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_230 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_42 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_74 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_258 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_167 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_201 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_92 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_602 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_638 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_468 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_42 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_0 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_18 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_301 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_18 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_26 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_505 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_349 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_136 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_392 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_613 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_685 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_65 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_9 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_808 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_760 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_273 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_0 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_704 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_397 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_171 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_275 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_738 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_661 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_73 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_562 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_508 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_743 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_141 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_222 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_795 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_316 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_547 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_686 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_453 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_205 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_507 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_93 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_527 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_200 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_397 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_354 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_174 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_210 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_465 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_230 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_352 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_763 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_318 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_761 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_608 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_732 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_327 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_80 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_301 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_394 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_0 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_108 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_562 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_394 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_743 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_690 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_553 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_286 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_795 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_352 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_78 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_175 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_495 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_602 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_174 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_386 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_200 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_230 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_549 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_266 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_127 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_284 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_135 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_603 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_462 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_738 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_643 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_495 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_504 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_511 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_720 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_42 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_202 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_652 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_462 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_73 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_135 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_47 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_175 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_243 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_316 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_545 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_465 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_106 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_316 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_746 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_508 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_557 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_141 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_95 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_199 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_63 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_631 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_26 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_237 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_122 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_242 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_161 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_699 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_736 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_714 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_690 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_160 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_784 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_78 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_101 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_507 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_136 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_213 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_375 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_565 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_475 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_349 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_199 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_108 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_74 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_643 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_787 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_136 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_273 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_346 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_273 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_258 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_341 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_720 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_345 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_191 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_9 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_237 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_103 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_505 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_318 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_125 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_222 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_130 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_167 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_667 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_130 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_54 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_468 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_74 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_395 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_52 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_395 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_418 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_106 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_205 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_45 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_104 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_171 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_603 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_242 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_275 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_38 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_101 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_557 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_494 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_596 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_104 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_343 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_553 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_238 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_657 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_596 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_763 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_167 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_456 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_602 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_69 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_324 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_171 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_26 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_219 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_324 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_613 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_714 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_659 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_52 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_549 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_659 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_345 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_689 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_80 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_324 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_392 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_275 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_343 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_284 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_456 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_760 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_184 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_689 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_395 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_9 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_457 !\n",
      "23/09/07 01:22:41 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_579 !\n",
      "23/09/07 01:22:50 ERROR TaskSchedulerImpl: Lost executor 2 on node55.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:50 WARN TaskSetManager: Lost task 24.0 in stage 17.0 (TID 5424) (node55.cluster.tsc.uc3m.es executor 2): ExecutorLostFailure (executor 2 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:50 WARN TaskSetManager: Lost task 14.0 in stage 17.0 (TID 5414) (node55.cluster.tsc.uc3m.es executor 2): ExecutorLostFailure (executor 2 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:50 WARN TaskSetManager: Lost task 4.0 in stage 17.0 (TID 5404) (node55.cluster.tsc.uc3m.es executor 2): ExecutorLostFailure (executor 2 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:50 WARN TaskSetManager: Lost task 34.0 in stage 17.0 (TID 5434) (node55.cluster.tsc.uc3m.es executor 2): ExecutorLostFailure (executor 2 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_123 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_49 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_495 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_531 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_481 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_186 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_580 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_599 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_63 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_46 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_89 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_365 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_456 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_388 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_271 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_118 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_783 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_31 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_54 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_265 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_534 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_124 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_490 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_440 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_702 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_354 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_180 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_326 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_630 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_238 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_436 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_224 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_123 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_170 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_124 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_204 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_197 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_329 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_47 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_438 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_285 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_329 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_210 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_283 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_46 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_781 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_204 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_615 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_5 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_266 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_728 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_31 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_329 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_585 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_365 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_364 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_249 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_66 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_197 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_161 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_285 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_354 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_91 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_681 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_721 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_685 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_482 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_337 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_320 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_266 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_153 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_685 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_561 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_804 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_639 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_525 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_155 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_580 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_155 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_49 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_224 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_217 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_82 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_613 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_118 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_196 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_152 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_769 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_31 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_562 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_89 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_588 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_737 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_238 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_85 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_24 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_225 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_5 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_16 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_531 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_673 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_376 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_540 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_187 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_225 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_84 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_118 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_16 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_189 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_135 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_119 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_590 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_314 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_665 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_82 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_227 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_665 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_161 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_24 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_525 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_440 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_733 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_780 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_91 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_281 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_155 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_325 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_780 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_722 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_59 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_804 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_630 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_737 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_49 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_703 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_588 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_484 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_5 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_482 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_281 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_85 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_748 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_59 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_437 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_639 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_761 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_236 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_24 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_476 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_365 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_79 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_783 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_655 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_382 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_598 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_548 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_119 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_459 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_63 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_438 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_637 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_120 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_187 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_280 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_721 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_153 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_222 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_170 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_376 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_81 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_519 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_356 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_236 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_599 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_761 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_702 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_484 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_117 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_690 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_319 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_722 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_655 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_436 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_280 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_153 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_540 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_650 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_293 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_320 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_506 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_16 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_194 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_196 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_490 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_421 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_319 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_548 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_443 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_795 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_364 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_598 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_437 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_476 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_97 !\n",
      "23/09/07 01:22:50 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_326 !\n",
      "23/09/07 01:23:03 ERROR TaskSchedulerImpl: Lost executor 9 on node23.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:03 WARN TaskSetManager: Lost task 18.0 in stage 17.0 (TID 5418) (node23.cluster.tsc.uc3m.es executor 9): ExecutorLostFailure (executor 9 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:03 WARN TaskSetManager: Lost task 8.0 in stage 17.0 (TID 5408) (node23.cluster.tsc.uc3m.es executor 9): ExecutorLostFailure (executor 9 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:03 WARN TaskSetManager: Lost task 38.0 in stage 17.0 (TID 5438) (node23.cluster.tsc.uc3m.es executor 9): ExecutorLostFailure (executor 9 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:03 WARN TaskSetManager: Lost task 28.0 in stage 17.0 (TID 5428) (node23.cluster.tsc.uc3m.es executor 9): ExecutorLostFailure (executor 9 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_452 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_749 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_778 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_683 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_452 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_83 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_139 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_701 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_231 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_644 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_98 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_790 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_378 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_689 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_36 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_379 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_212 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_804 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_98 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_128 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_555 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_582 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_517 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_672 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_467 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_83 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_544 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_684 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_713 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_479 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_586 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_139 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_379 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_753 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_128 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_644 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_656 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_407 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_627 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_753 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_34 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_639 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_749 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_706 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_544 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_35 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_212 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_735 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_634 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_121 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_656 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_734 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_407 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_749 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_576 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_19 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_313 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_34 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_121 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_264 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_19 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_525 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_121 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_576 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_735 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_467 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_35 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_505 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_188 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_797 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_523 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_698 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_683 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_516 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_623 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_672 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_467 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_192 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_340 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_684 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_361 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_378 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_570 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_98 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_706 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_501 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_464 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_568 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_691 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_609 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_699 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_609 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_139 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_509 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_623 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_779 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_565 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_583 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_626 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_691 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_753 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_497 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_452 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_568 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_192 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_231 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_714 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_774 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_734 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_501 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_706 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_770 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_361 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_701 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_774 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_648 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_565 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_775 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_734 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_83 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_361 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_476 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_634 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_778 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_36 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_790 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_621 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_264 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_221 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_713 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_190 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_313 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_593 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_568 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_190 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_698 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_340 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_188 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_779 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_479 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_313 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_790 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_644 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_621 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_740 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_464 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_701 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_593 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_625 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_570 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_625 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_648 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_264 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_462 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_583 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_221 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_699 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_340 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_559 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_407 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_34 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_707 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_501 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_691 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_647 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_740 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_523 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_188 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_516 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_797 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_35 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_509 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_797 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_570 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_521 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_192 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_623 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_19 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_212 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_36 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_672 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_190 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_221 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_231 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_609 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_684 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_737 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_698 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_774 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_378 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_634 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_379 !\n",
      "23/09/07 01:23:03 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_740 !\n",
      "23/09/07 01:23:21 ERROR TaskSchedulerImpl: Lost executor 3 on node51.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:21 WARN TaskSetManager: Lost task 21.0 in stage 17.0 (TID 5421) (node51.cluster.tsc.uc3m.es executor 3): ExecutorLostFailure (executor 3 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:21 WARN TaskSetManager: Lost task 11.0 in stage 17.0 (TID 5411) (node51.cluster.tsc.uc3m.es executor 3): ExecutorLostFailure (executor 3 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:21 WARN TaskSetManager: Lost task 31.0 in stage 17.0 (TID 5431) (node51.cluster.tsc.uc3m.es executor 3): ExecutorLostFailure (executor 3 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:21 WARN TaskSetManager: Lost task 1.0 in stage 17.0 (TID 5401) (node51.cluster.tsc.uc3m.es executor 3): ExecutorLostFailure (executor 3 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_146 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_355 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_146 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_327 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_720 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_127 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_253 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_621 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_189 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_626 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_487 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_119 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_624 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_697 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_668 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_253 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_319 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_767 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_489 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_683 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_743 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_206 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_626 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_39 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_322 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_283 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_453 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_39 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_151 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_429 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_453 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_681 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_1 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_483 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_582 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_328 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_578 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_451 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_64 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_75 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_383 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_267 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_243 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_186 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_771 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_214 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_27 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_739 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_145 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_587 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_534 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_487 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_532 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_12 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_739 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_446 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_385 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_658 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_767 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_12 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_587 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_472 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_665 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_695 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_20 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_457 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_227 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_388 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_242 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_715 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_530 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_504 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_771 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_215 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_256 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_306 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_294 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_574 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_209 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_339 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_189 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_294 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_801 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_353 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_20 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_744 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_105 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_215 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_535 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_544 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_1 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_65 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_670 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_364 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_658 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_667 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_105 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_37 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_363 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_85 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_760 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_342 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_323 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_111 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_492 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_532 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_27 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_162 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_388 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_545 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_631 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_719 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_120 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_631 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_236 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_729 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_95 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_39 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_450 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_799 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_182 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_780 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_228 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_90 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_250 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_504 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_162 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_363 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_38 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_725 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_744 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_624 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_695 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_616 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_763 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_791 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_584 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_90 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_582 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_534 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_658 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_75 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_267 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_1 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_205 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_591 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_151 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_160 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_619 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_122 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_43 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_472 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_243 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_181 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_183 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_110 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_160 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_132 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_175 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_299 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_545 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_322 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_725 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_218 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_38 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_250 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_12 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_182 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_458 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_143 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_645 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_283 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_532 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_342 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_132 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_465 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_68 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_355 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_259 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_120 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_612 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_301 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_496 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_483 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_416 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_64 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_37 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_327 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_294 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_186 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_397 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_579 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_306 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_282 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_67 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_668 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_499 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_286 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_348 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_43 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_149 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_791 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_417 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_67 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_151 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_94 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_695 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_64 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_616 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_92 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_206 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_788 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_86 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_127 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_719 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_90 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_179 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_227 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_286 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_549 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_72 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_446 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_223 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_579 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_179 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_20 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_508 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_788 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_670 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_588 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_40 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_61 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_122 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_578 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_681 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_267 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_348 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_383 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_530 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_496 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_211 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_429 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_65 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_197 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_457 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_697 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_176 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_383 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_211 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_218 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_27 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_46 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_335 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_416 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_124 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_95 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_92 !\n",
      "23/09/07 01:23:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_323 !\n",
      "23/09/07 01:23:28 ERROR TaskSchedulerImpl: Lost executor 0 on node28.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:28 WARN TaskSetManager: Lost task 0.0 in stage 17.0 (TID 5400) (node28.cluster.tsc.uc3m.es executor 0): ExecutorLostFailure (executor 0 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:28 WARN TaskSetManager: Lost task 30.0 in stage 17.0 (TID 5430) (node28.cluster.tsc.uc3m.es executor 0): ExecutorLostFailure (executor 0 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:28 WARN TaskSetManager: Lost task 20.0 in stage 17.0 (TID 5420) (node28.cluster.tsc.uc3m.es executor 0): ExecutorLostFailure (executor 0 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:28 WARN TaskSetManager: Lost task 10.0 in stage 17.0 (TID 5410) (node28.cluster.tsc.uc3m.es executor 0): ExecutorLostFailure (executor 0 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_390 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_442 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_794 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_517 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_577 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_660 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_696 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_688 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_784 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_529 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_550 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_606 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_11 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_488 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_536 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_696 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_564 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_592 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_432 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_109 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_765 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_561 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_756 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_669 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_30 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_389 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_566 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_87 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_10 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_56 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_765 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_769 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_478 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_551 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_693 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_809 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_791 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_723 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_498 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_87 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_577 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_510 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_439 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_393 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_663 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_593 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_585 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_669 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_116 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_496 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_769 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_148 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_543 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_163 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_581 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_109 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_660 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_178 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_109 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_390 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_705 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_786 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_96 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_372 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_198 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_605 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_11 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_115 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_500 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_138 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_419 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_386 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_550 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_628 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_463 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_450 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_522 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_420 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_646 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_531 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_601 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_178 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_198 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_660 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_2 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_446 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_592 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_491 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_738 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_663 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_2 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_803 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_362 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_386 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_373 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_733 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_338 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_522 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_628 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_131 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_458 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_551 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_528 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_178 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_581 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_693 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_805 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_551 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_700 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_96 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_498 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_777 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_298 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_455 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_338 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_419 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_148 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_30 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_392 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_493 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_601 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_344 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_116 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_420 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_450 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_355 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_730 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_168 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_168 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_423 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_702 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_421 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_442 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_429 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_131 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_764 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_561 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_575 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_594 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_528 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_481 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_491 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_700 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_439 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_518 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_500 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_315 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_420 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_805 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_10 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_610 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_269 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_380 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_705 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_786 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_454 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_722 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_794 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_298 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_56 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_455 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_688 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_718 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_669 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_317 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_705 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_803 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_628 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_356 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_649 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_723 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_431 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_0 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_10 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_794 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_373 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_493 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_655 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_688 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_756 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_630 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_727 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_11 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_700 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_617 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_390 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_463 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_784 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_723 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_163 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_148 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_303 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_358 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_589 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_389 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_430 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_517 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_432 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_372 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_581 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_765 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_116 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_115 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_786 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_663 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_96 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_356 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_485 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_482 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_755 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_315 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_137 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_776 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_638 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_269 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_131 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_241 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_783 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_269 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_809 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_431 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_675 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_730 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_566 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_500 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_478 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_661 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_592 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_470 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_756 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_682 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_498 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_431 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_137 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_380 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_315 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_396 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_529 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_241 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_727 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_510 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_733 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_661 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_755 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_589 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_198 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_536 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_419 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_675 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_646 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_439 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_423 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_776 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_137 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_543 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_788 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_617 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_455 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_767 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_10 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_488 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_389 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_493 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_718 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_138 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_536 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_2 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_348 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_33 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_317 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_138 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_522 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_567 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_577 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_396 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_543 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_585 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_693 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_423 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_374 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_805 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_241 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_564 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_564 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_809 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_636 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_33 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_488 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_636 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_776 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_168 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_30 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_87 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_553 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_56 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_430 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_739 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_303 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_317 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_362 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_646 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_303 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_638 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_298 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_432 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_115 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_533 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_481 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_33 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_605 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_338 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_718 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_128 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_372 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_163 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_421 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_466 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_430 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_380 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_777 !\n",
      "23/09/07 01:23:28 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_458 !\n",
      "23/09/07 01:23:37 ERROR TaskSchedulerImpl: Lost executor 5 on node65.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:37 WARN TaskSetManager: Lost task 36.0 in stage 17.0 (TID 5436) (node65.cluster.tsc.uc3m.es executor 5): ExecutorLostFailure (executor 5 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:37 WARN TaskSetManager: Lost task 6.0 in stage 17.0 (TID 5406) (node65.cluster.tsc.uc3m.es executor 5): ExecutorLostFailure (executor 5 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:37 WARN TaskSetManager: Lost task 26.0 in stage 17.0 (TID 5426) (node65.cluster.tsc.uc3m.es executor 5): ExecutorLostFailure (executor 5 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:37 WARN TaskSetManager: Lost task 16.0 in stage 17.0 (TID 5416) (node65.cluster.tsc.uc3m.es executor 5): ExecutorLostFailure (executor 5 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_287 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_29 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_86 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_183 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_152 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_666 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_357 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_771 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_382 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_770 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_447 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_37 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_524 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_68 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_318 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_448 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_497 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_88 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_557 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_134 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_292 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_716 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_398 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_624 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_377 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_451 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_113 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_643 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_674 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_574 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_26 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_381 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_328 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_335 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_764 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_618 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_67 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_88 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_486 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_117 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_594 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_438 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_618 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_229 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_213 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_474 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_502 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_474 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_486 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_94 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_173 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_108 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_141 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_798 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_330 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_503 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_252 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_524 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_362 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_214 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_22 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_448 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_473 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_55 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_4 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_44 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_306 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_239 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_692 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_86 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_673 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_258 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_741 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_721 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_447 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_94 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_293 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_249 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_77 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_696 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_554 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_224 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_254 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_239 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_61 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_497 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_618 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_607 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_113 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_84 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_367 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_252 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_785 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_287 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_152 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_607 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_606 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_213 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_580 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_123 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_292 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_110 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_293 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_173 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_806 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_332 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_447 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_715 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_214 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_719 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_607 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_296 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_52 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_22 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_785 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_181 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_145 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_16 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_546 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_58 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_29 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_130 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_181 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_692 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_6 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_217 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_554 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_15 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_173 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_292 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_332 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_89 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_418 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_524 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_100 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_662 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_649 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_169 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_240 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_145 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_360 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_594 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_523 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_729 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_574 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_547 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_55 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_15 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_117 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_764 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_29 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_502 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_360 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_546 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_729 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_73 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_666 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_149 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_328 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_4 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_716 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_349 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_381 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_418 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_474 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_322 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_84 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_183 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_331 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_798 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_110 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_252 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_134 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_149 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_134 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_15 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_68 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_162 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_649 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_662 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_204 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_673 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_727 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_335 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_331 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_606 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_382 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_4 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_367 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_61 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_58 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_199 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_742 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_280 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_281 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_530 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_55 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_104 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_547 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_770 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_215 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_777 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_184 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_184 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_217 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_715 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_193 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_484 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_202 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_202 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_296 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_668 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_179 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_587 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_741 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_566 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_451 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_22 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_249 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_789 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_428 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_666 !\n",
      "23/09/07 01:23:37 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_254 !\n",
      "23/09/07 01:35:47 ERROR TaskSchedulerImpl: Lost executor 4 on node33.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:35:47 WARN TaskSetManager: Lost task 9.0 in stage 17.0 (TID 5409) (node33.cluster.tsc.uc3m.es executor 4): ExecutorLostFailure (executor 4 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:35:47 WARN TaskSetManager: Lost task 39.0 in stage 17.0 (TID 5439) (node33.cluster.tsc.uc3m.es executor 4): ExecutorLostFailure (executor 4 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:35:47 WARN TaskSetManager: Lost task 41.0 in stage 17.0 (TID 5477) (node33.cluster.tsc.uc3m.es executor 4): ExecutorLostFailure (executor 4 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:35:47 WARN TaskSetManager: Lost task 43.0 in stage 17.0 (TID 5479) (node33.cluster.tsc.uc3m.es executor 4): ExecutorLostFailure (executor 4 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_710 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_409 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_641 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_277 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_244 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_245 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_679 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_347 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_425 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_288 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_263 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_433 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_29 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_400 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_150 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_234 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_556 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_403 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_150 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_159 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_404 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_676 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_412 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_512 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_308 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_435 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_156 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_424 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_408 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_444 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_9 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_310 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_414 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_571 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_402 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_808 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_14 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_409 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_572 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_425 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_773 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_514 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_434 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_411 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_426 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_520 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_276 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_308 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_262 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_279 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_477 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_158 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_114 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_404 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_514 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_799 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_678 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_405 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_750 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_807 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_410 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_150 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_436 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_800 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_276 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_573 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_513 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_263 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_444 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_773 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_157 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_408 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_808 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_263 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_556 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_569 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_289 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_114 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_515 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_677 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_424 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_758 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_7 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_772 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_247 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_614 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_251 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_642 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_515 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_757 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_633 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_235 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_571 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_403 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_369 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_370 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_410 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_802 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_572 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_262 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_311 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_289 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_632 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_772 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_480 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_512 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_614 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_426 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_29 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_261 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_757 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_233 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_310 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_611 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_260 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_640 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_279 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_632 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_717 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_537 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_537 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_751 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_246 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_404 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_520 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_571 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_290 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_251 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_412 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_800 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_676 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_751 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_641 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_279 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_157 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_520 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_289 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_538 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_807 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_677 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_751 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_234 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_573 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_633 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_405 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_334 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_632 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_234 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_640 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_611 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_251 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_159 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_773 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_368 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_158 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_401 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_290 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_569 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_572 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_260 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_402 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_278 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_710 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_278 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_370 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_278 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_406 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_435 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_291 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_157 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_359 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_368 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_539 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_308 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_802 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_233 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_371 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_247 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_539 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_757 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_277 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_403 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_515 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_310 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_246 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_288 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_427 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_641 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_291 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_401 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_412 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_676 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_537 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_750 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_614 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_232 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_538 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_246 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_232 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_311 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_750 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_405 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_437 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_244 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_413 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_512 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_232 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_14 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_678 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_400 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_717 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_7 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_514 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_427 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_410 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_556 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_633 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_369 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_477 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_731 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_807 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_539 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_309 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_759 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_408 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_260 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_759 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_635 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_262 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_368 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_513 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_710 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_731 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_245 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_480 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_406 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_288 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_758 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_569 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_679 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_401 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_159 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_235 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_311 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_276 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_635 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_370 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_409 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_371 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_414 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_411 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_114 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_573 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_334 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_800 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_235 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_347 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_538 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_359 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_156 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_413 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_480 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_39 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_611 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_261 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_640 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_334 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_635 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_513 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_717 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_402 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_642 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_415 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_291 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_803 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_772 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_759 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_19 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_247 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_434 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_277 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_347 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_369 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_425 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_424 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_427 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_371 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_413 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_731 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_309 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_802 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_406 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_411 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_677 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_156 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_758 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_233 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_679 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_19 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_14 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_678 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_426 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_261 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_444 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_158 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_309 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_359 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_7 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_433 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_400 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_642 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_244 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_434 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_415 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_799 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_477 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_245 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_433 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_290 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_17_415 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_6_435 !\n",
      "23/09/07 01:35:47 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_35_414 !\n",
      "23/09/07 02:14:21 ERROR TaskSchedulerImpl: Lost executor 14 on node12.cluster.tsc.uc3m.es: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 02:14:21 WARN TaskSetManager: Lost task 143.0 in stage 17.0 (TID 5583) (node12.cluster.tsc.uc3m.es executor 14): ExecutorLostFailure (executor 14 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 02:14:21 WARN TaskSetManager: Lost task 145.0 in stage 17.0 (TID 5585) (node12.cluster.tsc.uc3m.es executor 14): ExecutorLostFailure (executor 14 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 02:14:21 WARN TaskSetManager: Lost task 142.0 in stage 17.0 (TID 5582) (node12.cluster.tsc.uc3m.es executor 14): ExecutorLostFailure (executor 14 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 02:14:21 WARN TaskSetManager: Lost task 144.0 in stage 17.0 (TID 5584) (node12.cluster.tsc.uc3m.es executor 14): ExecutorLostFailure (executor 14 exited caused by one of the running tasks) Reason: Remote RPC client disassociated. Likely due to containers exceeding thresholds, or network issues. Check driver logs for WARN messages.\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_100 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_28 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_8 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_28 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_99 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_18 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_101 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_98 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_61 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_98 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_58 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_58 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_100 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_18 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_59 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_8 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_101 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_99 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_60 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_60 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_38 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_61 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_105_38 !\n",
      "23/09/07 02:14:21 WARN BlockManagerMasterEndpoint: No more replicas available for rdd_94_59 !\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in AI dataset (OpenAIRE): 2058409\n",
      "-RECORD 0-------------------------------------------------------------------------------------------------------------------------------\n",
      " id          | 4dc99724cf04::6dd4afb016bfc3cd91f6c05f94072f64                                                                           \n",
      " title       | Application of machine learning techniques for apple stock price prediction                                              \n",
      " description | The objective of this publication is to compare the level of prediction of certain techniques such as time series and... \n",
      " Kwd_count   | 3                                                                                                                        \n",
      "-RECORD 1-------------------------------------------------------------------------------------------------------------------------------\n",
      " id          | 4dc99724cf04::bff7674472f34ed68ae135a9c4be58a8                                                                           \n",
      " title       | Challenges of artificial intelligence in dentistry                                                                       \n",
      " description | Sr. Editor. Uno de los desafíos clave en la inteligencia artificial (IA) es cómo llevar el conocimiento informal a un... \n",
      " Kwd_count   | 1                                                                                                                        \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Matching with OpenAIRE publications\n",
    "\n",
    "result = spark.read.parquet(path2openAIRE + \"result\")\n",
    "papers_OA = result.filter(result[\"type\"] == \"publication\").select(\"id\").dropDuplicates()\n",
    "\n",
    "result_titles = spark.read.parquet(path2openAIRE + \"result_titles\").dropDuplicates(subset=[\"id\"])\n",
    "papers_OA = papers_OA.join(result_titles, on=\"id\", how=\"left\").drop(result_titles.id)\n",
    "\n",
    "result_description = spark.read.parquet(path2openAIRE + \"result_description\").dropDuplicates(subset=[\"id\"])\n",
    "papers_OA = papers_OA.join(result_description, on=\"id\", how=\"left\").drop(result_description.id).cache()\n",
    "\n",
    "AI_papers_OA = papers_OA.withColumn('raw_text', \n",
    "                    F.concat(F.col('title'),F.lit(' '), F.col('description')))\n",
    "\n",
    "AI_papers_OA = AI_papers_OA.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "AI_papers_OA = AI_papers_OA.drop(\"raw_text\").filter(AI_papers_OA.Kwd_count > 0).cache()\n",
    "\n",
    "print('Number of documents in AI dataset (OpenAIRE):', AI_papers_OA.count())\n",
    "AI_papers_OA.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6b1485c5-be3d-474e-b7fd-6c0a1f1e2d24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "AI_OA_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/OA_Kwds_AI.parquet\"\n",
    "AI_papers_OA.write.parquet(AI_OA_file)\n",
    "\n",
    "AI_OA_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/OA_Kwds3_AI.parquet\"\n",
    "AI_papers_OA.filter(AI_papers_OA.Kwd_count > 2).write.parquet(AI_OA_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6209109-00cc-440f-89ab-79446287e9c5",
   "metadata": {},
   "source": [
    "### 2.1.2. Patents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9f87a48c-e81e-4395-b3ee-7dd20140dbda",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/09/07 03:16:13 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
      "23/09/07 03:16:13 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
      "23/09/07 03:16:20 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.0\n",
      "23/09/07 03:16:20 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore jarenas@192.168.148.147\n",
      "23/09/07 03:16:20 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException\n",
      "23/09/07 03:16:20 WARN ObjectStore: Failed to get database parquet, returning NoSuchObjectException\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of patents with abstract in English language: 75348760\n",
      "root\n",
      " |-- appln_id: string (nullable = true)\n",
      " |-- docdb_family_id: string (nullable = true)\n",
      " |-- appln_title: string (nullable = true)\n",
      " |-- appln_title_lg: string (nullable = true)\n",
      " |-- appln_abstract: string (nullable = true)\n",
      " |-- appln_abstract_lg: string (nullable = true)\n",
      " |-- appln_filing_year: string (nullable = true)\n",
      " |-- earliest_filing_year: string (nullable = true)\n",
      " |-- granted: string (nullable = true)\n",
      " |-- appln_auth: string (nullable = true)\n",
      " |-- receiving_office: string (nullable = true)\n",
      " |-- ipr_type: string (nullable = true)\n",
      "\n",
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10000172                                                                                                                 \n",
      " docdb_family_id      | 5871993                                                                                                                  \n",
      " appln_title          | 2-Hydroxy-3-cyano-pyridone-(6) derivs - coupling compts. for the prepn. of azo dyes                                      \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The novel cpds. are of formula (I): (where R is (substd.)alkyl-, alkylene-, aralkyl-, cycloalkyl- or phenyl). They ar... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1973                                                                                                                     \n",
      " earliest_filing_year | 1973                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10000989                                                                                                                 \n",
      " docdb_family_id      | 5872463                                                                                                                  \n",
      " appln_title          | Sowing partially germinated seeds - by mixing seeds with fillers and transferring fillers and germinated seeds into soil \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | Seeds are sown in a manner reducing the germination time and in a weed free environment by mixing the seeds with fill... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1973                                                                                                                     \n",
      " earliest_filing_year | 1973                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "query = \"select appln_id, docdb_family_id, appln_title, appln_title_lg, appln_abstract, appln_abstract_lg, appln_filing_year, earliest_filing_year, \"\n",
    "query += \"granted, appln_auth, receiving_office, ipr_type from parquet.`\" + path2PATSTAT + \"/patstat_appln.parquet`\"\n",
    "df_patents = spark.sql(query)\n",
    "\n",
    "df_patents = df_patents.filter(F.col(\"appln_abstract\").isNotNull()).filter(df_patents.appln_abstract_lg==\"en\").cache()\n",
    "\n",
    "print('Number of patents with abstract in English language:', df_patents.count())\n",
    "df_patents.printSchema()\n",
    "df_patents.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e193224e-5ba5-4996-ba46-93b3ef6eb9e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/09/07 07:47:11 WARN CacheManager: Asked to cache already cached data.\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents in AI dataset (PATSTAT): 460820\n",
      "-RECORD 0----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 10512667                                                                                                                 \n",
      " docdb_family_id      | 5523418                                                                                                                  \n",
      " appln_title          | Metal foreign body detection for steel reinforced conveyor belt - compares properties of belt before and after delive... \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The detection of metal foreign bodies within material deposited on a steel wire reinforced conveyor belt is effected ... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1980                                                                                                                     \n",
      " earliest_filing_year | 1980                                                                                                                     \n",
      " granted              | N                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      " Kwd_count            | 1                                                                                                                        \n",
      "-RECORD 1----------------------------------------------------------------------------------------------------------------------------------------\n",
      " appln_id             | 11219486                                                                                                                 \n",
      " docdb_family_id      | 13134425                                                                                                                 \n",
      " appln_title          | Learning process for neural network - comparing sample signals and categorising for use in backward learning process     \n",
      " appln_title_lg       | en                                                                                                                       \n",
      " appln_abstract       | The learning process is used with a neural network that has an input layer (1) of 7 frames together with a further tw... \n",
      " appln_abstract_lg    | en                                                                                                                       \n",
      " appln_filing_year    | 1992                                                                                                                     \n",
      " earliest_filing_year | 1991                                                                                                                     \n",
      " granted              | Y                                                                                                                        \n",
      " appln_auth           | DE                                                                                                                       \n",
      " receiving_office     |                                                                                                                          \n",
      " ipr_type             | PI                                                                                                                       \n",
      " Kwd_count            | 2                                                                                                                        \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "AI_patents = df_patents.withColumn('raw_text', \n",
    "                    F.concat(F.col('appln_title'),F.lit(' '), F.col('appln_abstract')))\n",
    "\n",
    "AI_patents = AI_patents.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "AI_patents = AI_patents.drop(\"raw_text\").filter(AI_patents.Kwd_count > 0).cache()\n",
    "\n",
    "print('Number of documents in AI dataset (PATSTAT):', AI_patents.count())\n",
    "AI_patents.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1ad59c49-c08f-476a-af97-f8b0a49c31cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "AI_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/PATSTAT_Kwds_AI.parquet\"\n",
    "AI_patents.repartition(40).write.parquet(AI_file)\n",
    "\n",
    "AI_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/PATSTAT_Kwds3_AI.parquet\"\n",
    "AI_patents.filter(AI_patents.Kwd_count > 2).repartition(40).write.parquet(AI_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97c01720-e4d6-40d6-a7f4-d6b410b971cb",
   "metadata": {},
   "source": [
    "### 2.1.3. CORDIS projects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "346c096f-e9e6-4205-9779-50345f061b3c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/09/07 07:49:59 WARN package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects: 69613\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects with AI keywords: 4301\n",
      "-RECORD 0--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 218108                                                                                                                   \n",
      " acronym            | DHRS-CIM                                                                                                                 \n",
      " status             | CLO                                                                                                                      \n",
      " title              | Distributed Human-Robot System for Chemical Incident Management                                                          \n",
      " startDate          | 2008-10-01 00:00:00                                                                                                      \n",
      " endDate            | 2012-09-30 00:00:00                                                                                                      \n",
      " totalCost          | 1818833.0                                                                                                                \n",
      " ecMaxContribution  | 1818833.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-PEOPLE-2007-3-1-IAPP                                                                                                 \n",
      " fundingScheme      | MC-IAPP                                                                                                                  \n",
      " nature             | null                                                                                                                     \n",
      " objective          | The overall objective of this project is to combine forces of Europe’s leading industrial (Thales Nederland B.V., Spa... \n",
      " contentUpdateDate  | 2017-05-29 13:58:47                                                                                                      \n",
      " rcn                | 100615                                                                                                                   \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | PEOPLE-2007-3-1-IAPP                                                                                                     \n",
      " topic_title        | Marie Curie Action: \"Industry-Academia Partnerships and Pathways\"                                                        \n",
      " countryContr       | BE|296835.0 NL|418647.0 PT|164436.0 SE|553057.0 UK|385858.0                                                              \n",
      " orgContr           | 999757467.0|296835.0 999748155.0|164436.0 999842148.0|385858.0 999480241.0|302615.0 999650088.0|553057.0 999985708.0|... \n",
      " coordinatorCountry | NL                                                                                                                       \n",
      " coordinatorOrg     | 9.99480241E8                                                                                                             \n",
      " euroSciVocCode     | [1375]                                                                                                                   \n",
      " publicationID      | null                                                                                                                     \n",
      " patentID           | null                                                                                                                     \n",
      " Kwd_count          | 1                                                                                                                        \n",
      "-RECORD 1--------------------------------------------------------------------------------------------------------------------------------------\n",
      " projectID          | 231761                                                                                                                   \n",
      " acronym            | NoTube                                                                                                                   \n",
      " status             | CLO                                                                                                                      \n",
      " title              | NETWORKS AND ONTOLOGIES FOR THE TRANSFORMATION AND UNIFICATION OF BROADCASTING AND THE INTERNET                          \n",
      " startDate          | 2009-02-01 00:00:00                                                                                                      \n",
      " endDate            | 2012-01-31 00:00:00                                                                                                      \n",
      " totalCost          | 9249060.0                                                                                                                \n",
      " ecMaxContribution  | 6150000.0                                                                                                                \n",
      " ecSignatureDate    | null                                                                                                                     \n",
      " frameworkProgramme | FP7                                                                                                                      \n",
      " masterCall         | null                                                                                                                     \n",
      " subCall            | FP7-ICT-2007-3                                                                                                           \n",
      " fundingScheme      | CP                                                                                                                       \n",
      " nature             | null                                                                                                                     \n",
      " objective          | The goal of NoTube is to develop a service architecture, based on semantic technologies, for personalised creation, d... \n",
      " contentUpdateDate  | 2019-07-16 21:33:45                                                                                                      \n",
      " rcn                | 89494                                                                                                                    \n",
      " grantDoi           | null                                                                                                                     \n",
      " topic              | ICT-2007.4.4                                                                                                             \n",
      " topic_title        | Intelligent content and semantics (ICT-2007.4.4)                                                                         \n",
      " countryContr       | AT|498800.0 BG|376969.0 DE|548349.0 FR|562500.0 IT|1528778.0 KR|0.0 NL|1257004.0 TR|266400.0 UK|1111200.0                \n",
      " orgContr           | 997893806.0|376969.0 998184224.0|266400.0 999911988.0|562500.0 964558495.0|137634.0 954530344.0|759210.0 998307996.0|... \n",
      " coordinatorCountry | NL                                                                                                                       \n",
      " coordinatorOrg     | 9.54530344E8                                                                                                             \n",
      " euroSciVocCode     | [30060, 959]                                                                                                             \n",
      " publicationID      | null                                                                                                                     \n",
      " patentID           | null                                                                                                                     \n",
      " Kwd_count          | 2                                                                                                                        \n",
      "only showing top 2 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "projects = spark.read.parquet(path2cordis + \"projects.parquet\")\n",
    "print('Number of projects:', projects.count())\n",
    "\n",
    "AI_projects = projects.withColumn('raw_text', \n",
    "                    F.concat(F.col('title'),F.lit(' '), F.col('objective')))\n",
    "\n",
    "AI_projects = AI_projects.withColumn(\"Kwd_count\", count_kwds_UDF(F.col(\"raw_text\")))\n",
    "AI_projects = AI_projects.drop(\"raw_text\").filter(AI_projects.Kwd_count > 0).cache()\n",
    "\n",
    "print('Number of projects with AI keywords:', AI_projects.count())\n",
    "AI_projects.show(n=2, truncate=120, vertical=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4859c123-770b-4807-8995-db0db97156ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "AI_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/cordis_Kwds_AI.parquet\"\n",
    "AI_projects.repartition(40).write.parquet(AI_file)\n",
    "\n",
    "AI_file = \"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/cordis_Kwds3_AI.parquet\"\n",
    "AI_projects.filter(AI_projects.Kwd_count > 2).repartition(40).write.parquet(AI_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cb4149f-028f-43ef-94ad-fed026a5aec1",
   "metadata": {},
   "source": [
    "### 2.1.4. Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "06ced4e1-1840-4e76-8d20-6c633ecf3308",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAGHCAYAAAAZa5AYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABFwUlEQVR4nO3dd1gUV9sG8HvpiIAgSlEEbCiKIHZs2ECMLcZXE3uNRow1JhqjqDH29kUhajRiYhJLVKK+NhLFGqMo6GuNBQUFxYI0DW3P94fu6gooCzss7N6/6+JK9uzszDO7jHtz5swZmRBCgIiIiEgiBtougIiIiHQbwwYRERFJimGDiIiIJMWwQURERJJi2CAiIiJJMWwQERGRpBg2iIiISFIMG0RERCQphg0iIiKSFMMGSerChQsYOnQo3NzcYGZmhvLly8PHxweLFi3CkydPtF3eW82aNQsymaxIr927dy9mzZqV73Ourq4YMmRI0QvTIUOGDEH58uW1XUahbdmyBfXq1YO5uTlkMhliYmK0XVKpM2TIELi6ump0nfHx8RgzZgxq164Nc3Nz2NrawtPTEyNHjkR8fLxyuR07duCjjz5CzZo1YW5uDldXV/Tv3x/Xr1/XaD2kPiNtF0C66/vvv8eYMWPg7u6OKVOmwMPDA9nZ2YiKisLq1avx119/YefOndouUxJ79+5FSEhIvoFj586dsLKyKvmiqFgePnyIgQMHonPnzggNDYWpqSlq166t7bJKnRkzZmD8+PEaW9/du3fh4+ODChUqYPLkyXB3d0dKSgouX76MrVu34tatW3B2dgYALFy4EA4ODpg+fTqqV6+O+Ph4zJs3Dz4+Pjh16hTq1aunsbpIPQwbJIm//voLn3zyCTp16oTw8HCYmpoqn+vUqRMmT56M/fv3a7FC7WnYsKG2S9A7z549Q7ly5Yq1jn/++QfZ2dkYMGAA2rZtq6HKSp/ivlc1atTQYDUv/mh59OgRTp8+DTc3N2V7z5498eWXX0Iulyvbdu/ejcqVK6u8vn379nB1dcXy5cuxbt06jdZGhcfTKCSJefPmQSaTYe3atSpBQ8HExATdu3dXPpbJZPn2Arx5yiEsLAwymQyHDh3CyJEjUbFiRVhZWWHQoEHIyMjA/fv30adPH1SoUAGOjo747LPPkJ2drXx9ZGQkZDIZIiMjVbZz+/ZtyGQyhIWFvXW/tmzZAn9/fzg6OsLc3Bx169bF1KlTkZGRoVxmyJAhCAkJUe6X4uf27dt59unhw4cwMTHBjBkz8mzr6tWrkMlk+Pbbb5Vt9+/fx6hRo1C1alWYmJjAzc0Ns2fPRk5OzlvrVmy3a9eu2L9/P3x8fGBubo46derghx9+UFmuoNNHivdesR+vr3PPnj1o2LCh8j3Zs2eP8jV169aFhYUFmjZtiqioqHxru3TpEjp06AALCwtUqlQJY8eOxbNnz1SWEUIgNDQU3t7eMDc3h42NDXr37o1bt26pLOfn54f69evj6NGj8PX1Rbly5TBs2LC3vje7du1CixYtUK5cOVhaWqJTp07466+/lM8PGTIErVq1AgD07dsXMpkMfn5+Ba5P8V5FRERg6NChsLW1hYWFBbp165an3oiICPTo0QNVq1aFmZkZatasiVGjRuHRo0cqyyk+l+joaPTq1QtWVlawtrbGgAED8PDhwzw1bNmyBS1atICFhQXKly+PgIAAREdHqyyjOI31v//9D/7+/rC0tESHDh0AANHR0ejatSsqV64MU1NTODk54b333sPdu3ff+l7mdxpFJpNh7Nix+Omnn1C3bl2UK1cOXl5eyt+Tt3n8+DEMDAzyhAgFA4NXX2P5LePk5ISqVauqnG6hksewQRqXm5uLQ4cOoVGjRsruTU0bMWIErK2tsXnzZnz11Vf45ZdfMHLkSLz33nvw8vLCb7/9hsGDB2Pp0qVYuXKlxrZ7/fp1dOnSBevXr8f+/fsxYcIEbN26Fd26dVMuM2PGDPTu3RvAix4exY+jo2Oe9VWqVAldu3bFxo0bVf5CA4ANGzbAxMQE/fv3B/AiaDRt2hQHDhzAzJkzsW/fPgwfPhzz58/HyJEjC1X/+fPnMXnyZEycOBG///47GjRogOHDh+Po0aNFfUtw/vx5TJs2DV988QV27NgBa2tr9OrVC8HBwVi3bh3mzZuHn3/+GSkpKejatSueP3+u8vrs7Gx06dIFHTp0QHh4OMaOHYs1a9agb9++KsuNGjUKEyZMQMeOHREeHo7Q0FBcunQJvr6+ePDggcqyiYmJGDBgAPr164e9e/dizJgxBdb/yy+/oEePHrCyssKvv/6K9evXIzk5GX5+fjh+/DiAF5+pIkDOmzcPf/31F0JDQ9/53gwfPhwGBgb45ZdfsGLFCpw+fRp+fn54+vSpcpmbN2+iRYsW+O6773Dw4EHMnDkTf//9N1q1aqUSlBXef/991KxZE7/99htmzZqF8PBwBAQEqCw7b948fPTRR/Dw8MDWrVvx008/IS0tDa1bt8bly5dV1peVlYXu3bujffv2+P333zF79mxkZGSgU6dOePDgAUJCQhAREYEVK1agWrVqSEtLe+d+5+e///0vVq1ahTlz5mD79u2wtbXF+++/nyd8valFixaQy+Xo1asXDhw4gNTUVLW2e+vWLdy5c4enULRNEGnY/fv3BQDx4YcfFvo1AERwcHCedhcXFzF48GDl4w0bNggA4tNPP1VZrmfPngKAWLZsmUq7t7e38PHxUT4+fPiwACAOHz6sslxsbKwAIDZs2KBsCw4OFm87RORyucjOzhZHjhwRAMT58+eVzwUFBRX42jf3adeuXQKAOHjwoLItJydHODk5iQ8++EDZNmrUKFG+fHlx584dlfUtWbJEABCXLl0qsFbFds3MzFRe//z5c2FraytGjRr1zv1WvPexsbEq6zQ3Nxd3795VtsXExAgAwtHRUWRkZCjbw8PDBQCxa9cuZdvgwYMFAPF///d/Ktv65ptvBABx/PhxIYQQf/31lwAgli5dqrJcfHy8MDc3F59//rmyrW3btgKA+PPPP9/6fgghRG5urnBychKenp4iNzdX2Z6WliYqV64sfH19lW2K351t27a9c72K9+r9999XaT9x4oQAIObOnZvv6xS/U3fu3BEAxO+//658TvG5TJw4UeU1P//8swAgNm3aJIQQIi4uThgZGeU5RtLS0oSDg4Po06ePsk3x/v/www8qy0ZFRQkAIjw8/J37+qbBgwcLFxcXlTYAwt7eXqSmpirb7t+/LwwMDMT8+fPfuj65XC5GjRolDAwMBAAhk8lE3bp1xcSJE1V+F/OTnZ0t/Pz8hJWVlYiLi1N7X0hz2LNBZVLXrl1VHtetWxcA8N577+Vpv3Pnjsa2e+vWLfTr1w8ODg4wNDSEsbGx8vz9lStXirTOwMBAODg4YMOGDcq2AwcOICEhQaX7f8+ePWjXrh2cnJyQk5Oj/AkMDAQAHDly5J3b8vb2RrVq1ZSPzczMULt27WK9R97e3qhSpYryseKz8PPzUzn3r2jPb1uK3huFfv36AQAOHz4M4MW+y2QyDBgwQGXfHRwc4OXllee0mI2NDdq3b//O2q9du4aEhAQMHDhQpTu+fPny+OCDD3Dq1Kk8p3PU8eZ++fr6wsXFRblfAJCUlITRo0fD2dkZRkZGMDY2houLC4D8f6feXGefPn1gZGSkXOeBAweQk5ODQYMGqbxXZmZmaNu2bZ73CgA++OADlcc1a9aEjY0NvvjiC6xevTpPb0hRtGvXDpaWlsrH9vb2qFy58jt/92QyGVavXo1bt24hNDQUQ4cORXZ2NpYvX4569eoV+HsvhMDw4cNx7Ngx/Pjjj5L1slLhcIAoaZydnR3KlSuH2NhYybZha2ur8tjExKTA9n///Vcj20xPT0fr1q1hZmaGuXPnonbt2ihXrhzi4+PRq1evPKcHCsvIyAgDBw7EypUr8fTpU1SoUAFhYWFwdHREQECAcrkHDx5g9+7dMDY2znc9b57jz0/FihXztJmamha5dkC9zwJAns/DyMgoT10ODg4AXpyvB17suxAC9vb2+dZQvXp1lcf5nbLKj2L9+S3v5OQEuVyO5OTkIg+YVOzHm22K7crlcvj7+yMhIQEzZsyAp6cnLCwsIJfL0bx583w/lzfXqXj/Xn+vAKBJkyb51vR6qAKAcuXK5bk6ytraGkeOHME333yDL7/8EsnJyXB0dMTIkSPx1VdfFfg7+DbF/d1zcXHBJ598ony8detWfPTRR5gyZQpOnz6tsqwQAiNGjMCmTZuwceNG9OjRQ+16SbMYNkjjDA0N0aFDB+zbtw93795F1apV3/kaU1NTZGZm5mlX/AOqKWZmZgCQZ1uF+aI+dOgQEhISEBkZqXI1wuvn34tq6NChWLx4MTZv3oy+ffti165dmDBhAgwNDZXL2NnZoUGDBvjmm2/yXYeTk1Ox6wBU36PXB/cW5j0qipycHDx+/Fjly+j+/fsAXn1B2dnZQSaT4dixY/kOOH6zrbDzoyjWn5iYmOe5hIQEGBgYwMbGpnA7kg/FfrzZVrNmTQDAxYsXcf78eYSFhWHw4MHKZW7cuPHWdb7ek/Tm+2dnZwcA+O2335Q9JG9T0Hvl6emJzZs3QwiBCxcuICwsDHPmzIG5uTmmTp36zvVKrU+fPpg/fz4uXryo0q4IGhs2bMD69esxYMAALVVIr+NpFJLEtGnTIITAyJEjkZWVlef57Oxs7N69W/nY1dUVFy5cUFnm0KFDSE9P12hdilHyb25r165d73yt4h/lN7/Y1qxZk2dZxTKF/autbt26aNasGTZs2IBffvkFmZmZGDp0qMoyXbt2xcWLF1GjRg00btw4z4+mwkZB79Hrn5em/fzzzyqPf/nlFwBQXvHRtWtXCCFw7969fPfd09OzSNt1d3dHlSpV8Msvv0AIoWzPyMjA9u3blVeoFNWb+3Xy5EncuXNHuV/q/E4VtM6tW7ciJydHuc6AgAAYGRnh5s2b+b5XjRs3VmsfZDIZvLy8sHz5clSoUAHnzp1T6/XFlV8QBF70NMbHx6v83iv+zdmwYQPWrFmT5xgi7WHPBklCMbp+zJgxaNSoET755BPUq1cP2dnZiI6Oxtq1a1G/fn3lVRwDBw7EjBkzMHPmTLRt2xaXL1/GqlWrYG1trdG6HBwc0LFjR8yfPx82NjZwcXHBn3/+iR07drzztb6+vrCxscHo0aMRHBwMY2Nj/Pzzzzh//nyeZRVffgsXLkRgYCAMDQ3RoEED5amE/AwbNgyjRo1CQkICfH194e7urvL8nDlzEBERAV9fX4wbNw7u7u74999/cfv2bezduxerV68uVC/Su3Tp0gW2trYYPnw45syZAyMjI4SFhUl26aCJiQmWLl2K9PR0NGnSBCdPnsTcuXMRGBiovNy0ZcuW+PjjjzF06FBERUWhTZs2sLCwQGJiIo4fPw5PT0+VLvbCMjAwwKJFi9C/f3907doVo0aNQmZmJhYvXoynT59iwYIFxdq3qKgojBgxAv/5z38QHx+P6dOno0qVKsqrY+rUqYMaNWpg6tSpEELA1tYWu3fvRkRERIHr3LFjB4yMjNCpUydcunQJM2bMgJeXF/r06QPgRVicM2cOpk+fjlu3bqFz586wsbHBgwcPcPr0aVhYWGD27NlvrXvPnj0IDQ1Fz549Ub16dQghsGPHDjx9+hSdOnUq1nuirm+++QYnTpxA3759lZc9x8bGYtWqVXj8+DEWL16sXHbcuHFYv349hg0bBk9PT5w6dUr5nKmpKee40SYtDUwlPRETEyMGDx4sqlWrJkxMTISFhYVo2LChmDlzpkhKSlIul5mZKT7//HPh7OwszM3NRdu2bUVMTEyBV6OcOXNGZTuKkfoPHz5UaR88eLCwsLBQaUtMTBS9e/cWtra2wtraWgwYMEA5+v5dV6OcPHlStGjRQpQrV05UqlRJjBgxQpw7dy7PazMzM8WIESNEpUqVhEwmU7mK4819UkhJSRHm5uYCgPj+++/zfT8fPnwoxo0bJ9zc3ISxsbGwtbUVjRo1EtOnTxfp6en5vkbBxcVFvPfee3na27ZtK9q2bavSdvr0aeHr6yssLCxElSpVRHBwsFi3bl2+V6Pkt04AIigoSKVNccXP4sWLlW2Kz+fChQvCz89PmJubC1tbW/HJJ5/kuz8//PCDaNasmbCwsBDm5uaiRo0aYtCgQSIqKkplf+rVq/fW9+JN4eHholmzZsLMzExYWFiIDh06iBMnTqgsU5SrUQ4ePCgGDhwoKlSoIMzNzUWXLl3E9evXVZa9fPmy6NSpk7C0tBQ2NjbiP//5j4iLi8tzhZbi9/Hs2bOiW7duonz58sLS0lJ89NFH4sGDB/nuU7t27YSVlZUwNTUVLi4uonfv3uKPP/5QLpPf8SGEEFevXhUfffSRqFGjhjA3NxfW1taiadOmIiws7J37XtDVKG/+PghR8LHwulOnTomgoCDh5eUlbG1thaGhoahUqZLo3Lmz2Lt3b571Acj3582aqGTJhHit75CIiIotLCwMQ4cOxZkzZ9Q+bVGQWbNmYfbs2Xj48KFyXAZRWcExG0RERCQphg0iIiKSFE+jEBERkaTYs0FERESSYtggIiIiSTFsEBERkaT0flIvuVyOhIQEWFpaFnqKYyIiInoxa2taWhqcnJzy3HfndXofNhISEng3QCIiomKIj49/6wzGeh82FLc8jo+Pz3PnQyIiIipYamoqnJ2dld+lBdH7sKE4dWJlZcWwQUREVATvGobAAaJEREQkKYYNIiIikhTDBhEREUmKYYOIiIgkxbBBREREkmLYICIiIkkxbBAREZGkGDaIiIhIUgwbREREJCm9n0GUiIhI1128l4K7yc9Qs3J51Kz89qnFpcCeDSIiIh336+k4jN50Dv+9cF8r22fYICIiIkkxbBAREZGk9DZshISEwMPDA02aNNF2KURERDpNb8NGUFAQLl++jDNnzmi7FCIiIkkJLW9fb8MGERGRvpHJtLNdhg0iIiKSFMMGERERSYphg4iIiCTFsEFERKTjhJZHiDJsEBERkaQYNoiIiPSEli5GYdggIiIiaTFsEBERkaQYNoiIiHSedkeIMmwQERGRpBg2iIiI9ASnKyciIiKdxLBBREREkmLYICIi0nGcQZSIiIh0GsMGERGRnpBpaYQowwYRERFJimGDiIiIJMWwQUREpOM4QJSIiIh0GsMGERERSYphg4iIiCTFsEFERESSYtggIiLScYK3mCciIiJdxrBBRESkJ3iLeSIiItJJDBtEREQkqTIfNuLj4+Hn5wcPDw80aNAA27Zt03ZJREREpYq2ZxA10u7mi8/IyAgrVqyAt7c3kpKS4OPjgy5dusDCwkLbpRERERF0IGw4OjrC0dERAFC5cmXY2triyZMnDBtERERvkEFPbzF/9OhRdOvWDU5OTpDJZAgPD8+zTGhoKNzc3GBmZoZGjRrh2LFj+a4rKioKcrkczs7OEldNREREhaX1sJGRkQEvLy+sWrUq3+e3bNmCCRMmYPr06YiOjkbr1q0RGBiIuLg4leUeP36MQYMGYe3atW/dXmZmJlJTU1V+iIiISDpaDxuBgYGYO3cuevXqle/zy5Ytw/DhwzFixAjUrVsXK1asgLOzM7777jvlMpmZmXj//fcxbdo0+Pr6vnV78+fPh7W1tfKHvSBERKTrtDw+VPth422ysrJw9uxZ+Pv7q7T7+/vj5MmTAAAhBIYMGYL27dtj4MCB71zntGnTkJKSovyJj4+XpHYiIiJ6oVQPEH306BFyc3Nhb2+v0m5vb4/79+8DAE6cOIEtW7agQYMGyvEeP/30Ezw9PfNdp6mpKUxNTSWtm4iIiF4p1WFDQfbG/KpCCGVbq1atIJfLtVEWERFRmcLpyvNhZ2cHQ0NDZS+GQlJSUp7eDiIiIiqdSnXYMDExQaNGjRAREaHSHhER8c6BoERERPSC3s8gmp6ejhs3bigfx8bGIiYmBra2tqhWrRomTZqEgQMHonHjxmjRogXWrl2LuLg4jB49uljbDQkJQUhICHJzc4u7C0RERPQWWg8bUVFRaNeunfLxpEmTAACDBw9GWFgY+vbti8ePH2POnDlITExE/fr1sXfvXri4uBRru0FBQQgKCkJqaiqsra2LtS4iIiIqmNbDhp+fH8Q7+nfGjBmDMWPGlFBFREREuklL40NL95gNIiIiKvsYNoiIiHSc0PIcogwbREREJCm9DRshISHw8PBAkyZNtF0KERGRTtPbsBEUFITLly/jzJkz2i6FiIioRHAGUSIiItJJDBtERES6TssziDJsEBERkaQYNoiIiEhSDBtERER6QqalOUT1Nmzw0lciIqKSobdhg5e+EhGRvtDy+FD9DRtERERUMhg2iIiISFIMG0RERHqCM4gSERGRTmLYICIi0nFC8BbzREREpMP0Nmxwng0iIqKSobdhg/NsEBERlQy9DRtERERUMhg2iIiIdJxOzCD69OlTTayGiIiIdJDaYWPhwoXYsmWL8nGfPn1QsWJFVKlSBefPn9docURERFT2qR021qxZA2dnZwBAREQEIiIisG/fPgQGBmLKlCkaL5CIiIjKNiN1X5CYmKgMG3v27EGfPn3g7+8PV1dXNGvWTOMFEhERkWbItDRfudo9GzY2NoiPjwcA7N+/Hx07dgTwYnay3NxczVZHRERExablCUTV79no1asX+vXrh1q1auHx48cIDAwEAMTExKBmzZoaL5CIiIjKNrXDxvLly+Hq6or4+HgsWrQI5cuXB/Di9MqYMWM0XqBUQkJCEBISwt4YIiIiicmEtu/OomWpqamwtrZGSkoKrKystF0OERGRxo37NRq7zidgRlcPDG/lprH1FvY7tFA9G7t27Sr0hrt3717oZYmIiKjkaGd4aCHDRs+ePVUey2QyldvVvj66lacliIiIShdtn8Io1NUocrlc+XPw4EF4e3tj3759ePr0KVJSUrB37174+Phg//79UtdLREREZYzaA0QnTJiA1atXo1WrVsq2gIAAlCtXDh9//DGuXLmi0QKJiIiobFN7no2bN2/C2to6T7u1tTVu376tiZqIiIhIh6gdNpo0aYIJEyYgMTFR2Xb//n1MnjwZTZs21WhxREREVHyKcZZamkBU/bCxfv16JCUlwcXFBTVr1kTNmjVRrVo1JCYmYv369VLUSERERGWY2mM2atWqhfPnz+OPP/7A1atXIYSAh4cHOnbsqLU514mIiKj0Uits5OTkwMzMDDExMfD394e/v79UdREREZGOUOs0ipGREVxcXHRiLo2QkBB4eHigSZMm2i6FiIhIp6k9ZuOrr77CtGnT8OTJEynqKTFBQUG4fPkyzpw5o+1SiIiIJKWY1KtUzyD6um+//RY3btyAk5MTXFxcYGFhofL8uXPnNFYcERERlX1qh403py4nIiIiehu1w0ZwcLAUdRAREZGOUjtsKJw9exZXrlyBTCaDh4cHGjZsqMm6iIiISEeoHTaSkpLw4YcfIjIyEhUqVIAQAikpKWjXrh02b96MSpUqSVEnERERFdXLEaLamg9L7atRPv30U6SmpuLSpUt48uQJkpOTcfHiRaSmpmLcuHFS1EhERERlmNo9G/v378cff/yBunXrKts8PDwQEhLCSb6IiIgoD7V7NuRyOYyNjfO0GxsbQy6Xa6QoIiIi0h1qh4327dtj/PjxSEhIULbdu3cPEydORIcOHTRaHBEREZV9aoeNVatWIS0tDa6urqhRowZq1qwJNzc3pKWlYeXKlVLUSERERMUglHOIaofaYzacnZ1x7tw5RERE5LnrKxEREZVe2ro5u9ph49mzZyhXrhw6deqETp06SVETERER6RC1w0aFChXQuHFj+Pn5wc/PDy1btsxzfxQiIiIiBbXHbBw5cgTdu3fHuXPn0Lt3b9jY2KB58+aYOnUq9u3bJ0WNkuAt5omIiEqGTAhR5FEjubm5OHPmDFavXo2ff/4Zcrkcubm5mqxPcqmpqbC2tkZKSgqsrKy0XQ4REZHGfbLpLPZdvI85PephUAtXja23sN+hRbo3ytWrVxEZGYkjR44gMjIS2dnZ6NatG9q2bVvkgomIiEhaWhofqn7YcHBwQHZ2Ntq3bw8/Pz98+eWX8PT0lKI2IiIi0gFqj9lwcHBAeno64uLiEBcXh7t37yI9PV2K2oiIiEgHqB02YmJi8ODBA0yfPh05OTmYMWMGKlWqhGbNmmHq1KlS1EhERERlWJHGbFSoUAHdu3dHq1at0LJlS/z+++/45ZdfEBUVhQULFmi6RiIiIiqGol8Kohlqh42dO3ciMjISkZGRuHTpEipWrIjWrVtj+fLlaNeunRQ1EhERkSZoaQpRtcPGqFGj0KZNG4wcORJ+fn6oX7++FHURERGRjlA7bCQlJUlRBxEREekotQeI+vn54ccff8Tz58+lqIeIiIh0jNpho1GjRvj888/h4OCAkSNH4tSpU1LURURERBqi7VvMqx02li5dinv37uHHH3/Ew4cP0aZNG3h4eGDJkiV48OCBFDUSERGRBmhrBlG1wwYAGBoaokePHggPD8e9e/fQr18/zJgxA87OzujZsycOHTqk6TqJiIiojCpS2FA4ffo0Zs6ciSVLlqBy5cqYNm0aKleujG7duuGzzz7TVI1ERERUhhXpapSffvoJGzZswPXr19GtWzds3rwZAQEBkL28frdPnz7o2bMnlixZovGCiYiISD3Ps+Va3b7aYaNq1aqoUaMGhg0bhiFDhqBSpUp5lmnatCmaNGmikQKJiIioeO4lPwMAZOdqJ3SoHTb+/PNPtG7d+q3LWFlZ4fDhw0UuioiIiDTHtaIFbj7MwPPsXK1sX+0xG61bt0ZOTg7++OMPrFmzBmlpaQCAhIQE3v2ViIioFMp9eXOUSuVNtbJ9tXs27ty5g86dOyMuLg6ZmZno1KkTLC0tsWjRIvz7779YvXq1FHUSERFREclfTrNhaKCdi1/V7tkYP348GjdujOTkZJibmyvb33//ffz5558aLY6IiIiKT/4ybRiUlRuxHT9+HCdOnICJiYlKu4uLC+7du6exwoiIiEgzchVho6z0bMjlcuTm5h1gcvfuXVhaWmqkqJIQEhICDw8PXjVDREQ6T/5yzIahlno21A4bnTp1wooVK5SPZTIZ0tPTERwcjC5dumiyNkkFBQXh8uXLOHPmjLZLISIikpQibGipY0P90yjLly9Hu3bt4OHhgX///Rf9+vXD9evXYWdnh19//VWKGomIiKgYtH0aRe2w4eTkhJiYGGzevBlnz56FXC7H8OHD0b9/f5UBo0RERFQ6KK9GKSsDRDdt2oQBAwZg6NChGDp0qMpzU6ZMweLFizVWHBERERVfTPxTAIBBse6IVnRqb3bs2LHYs2dPnvaJEydi06ZNGimKiIiINMfNzgIAkJWjnenK1Q4bmzdvxoABA3D06FFl26effoqtW7dyinIiIqJSSLwcIGqnpRlE1Q4bnTt3xurVq9GzZ09ERUVhzJgx2LFjBw4fPow6depIUSMREREVQ3buy0tfy8oAUQD48MMPkZycjFatWqFSpUo4cuQIatasqenaiIiISAMUV6MYG2pn0EahwsakSZPyba9cuTIaNmyI0NBQZduyZcs0UxkRERFpRI78xViNUt2zER0dnW97jRo1kJqaqnxepqVLaoiIiKhgj9KzAADGhqU4bHDgJxERUdmnrRuxaemKWyIiIiopitMnFqZFGqpZbAwbREREOiw7V64cIGpmZKiVGhg2iIiIdFj6vznK/zc11s7XPsMGERGRDkvPfBU2zIxLcc+Gj48PkpOTAQBz5szBs2fPJC2KiIiINCMzJxcAYFPOWGs1FCpsXLlyBRkZGQCA2bNnIz09XdKiiIiISDOSn2UDAEy1NF4DKOSlr97e3hg6dChatWoFIQSWLFmC8uXL57vszJkzNVogERERFd3NpBcdBCnPs7VWQ6HCRlhYGIKDg7Fnzx7IZDLs27cPRkZ5XyqTyRg2iIiISqEqNuZa23ahwoa7uzs2b94MADAwMMCff/6JypUrS1oYERERFV9W7oupymvb539GoiSoPbuH/OX86kRERFT6Hbn2EABgoqWbsAFFvOvrzZs3sWLFCly5cgUymQx169bF+PHjUaNGDU3XR0RERMWQ9nKejfTMXK3VoHbMOXDgADw8PHD69Gk0aNAA9evXx99//4169eohIiJCihqJiIioiOTixeyh3bwctVaD2j0bU6dOxcSJE7FgwYI87V988QU6deqkseKIiIioeLJfjtkor6X7ogBF6Nm4cuUKhg8fnqd92LBhuHz5skaKIiIiouLLlQucv5sCADAx0t6YDbW3XKlSJcTExORpj4mJ4RUqREREpcjp2CfK/7cpZ6K1OtTuUxk5ciQ+/vhj3Lp1C76+vpDJZDh+/DgWLlyIyZMnS1EjERERFcHrE3nVc7LSWh1qh40ZM2bA0tISS5cuxbRp0wAATk5OmDVrFsaNG6fxAomIiKhoFOM1mle3hUwm01odaocNmUyGiRMnYuLEiUhLSwMAWFpaarwwIiIiKh5F2DDW4hwbQDFvMW9paVkqgsb7778PGxsb9O7dW9ulEBERlQr/Zudi0tbzAABTLQ4OBYoZNkqLcePG4ccff9R2GURERKXG7ccZyv9v4mqrxUp0JGy0a9euVPSwEBERlRbZOS8m8ypnYohRbbU7w7fWw8bRo0fRrVs3ODk5QSaTITw8PM8yoaGhcHNzg5mZGRo1aoRjx46VfKFERERliOIGbHblTbVciZphIzs7G+3atcM///yjsQIyMjLg5eWFVatW5fv8li1bMGHCBEyfPh3R0dFo3bo1AgMDERcXp7EaiIiIdE3Oy7BhZKi9q1AU1LoaxdjYGBcvXtTo5TOBgYEIDAws8Plly5Zh+PDhGDFiBABgxYoVOHDgAL777jvMnz9f7e1lZmYiMzNT+Tg1NVX9oomIiEqxEzceof+6vwFo926vCmpXMGjQIKxfv16KWvLIysrC2bNn4e/vr9Lu7++PkydPFmmd8+fPh7W1tfLH2dlZE6USERGVGseuP1L+fzM37Q4OBYowz0ZWVhbWrVuHiIgING7cGBYWFirPL1u2TGPFPXr0CLm5ubC3t1dpt7e3x/3795WPAwICcO7cOWRkZKBq1arYuXMnmjRpku86p02bhkmTJikfp6amMnAQEZFOUcyv8WETZ8zuUV/L1RQhbFy8eBE+Pj4AkGfshlSzk725XiGEStuBAwcKvS5TU1OYmmp/sAwREZFUckrR4FCgCGHj8OHDUtSRLzs7OxgaGqr0YgBAUlJSnt4OIiIieiFb/uKy19IwOBQoxqWvN27cwIEDB/D8+XMAL3obNM3ExASNGjVCRESESntERAR8fX01vj0iIqKyLCn1X6w9ehP/e3lbeW1PU66gds/G48eP0adPHxw+fBgymQzXr19H9erVMWLECFSoUAFLly5Va33p6em4ceOG8nFsbCxiYmJga2uLatWqYdKkSRg4cCAaN26MFi1aYO3atYiLi8Po0aPVLZ2IiEinLf/jH/x6Ol752NJM7a95SagdeSZOnAhjY2PExcWhXLlyyva+ffti//79ahcQFRWFhg0bomHDhgCASZMmoWHDhpg5c6ZyvStWrMCcOXPg7e2No0ePYu/evXBxcVF7W68LCQmBh4dHgQNJiYiIyponGVkAgMYuNhjVtjp6eFXRckUvyISa5z8cHBxw4MABeHl5wdLSEufPn0f16tURGxsLT09PpKenS1WrJFJTU2FtbY2UlBRYWVlpuxwiIqIiG7HxDP64koQFvTzxYdNqkm+vsN+havdsZGRkqPRoKDx69IhXeRAREWlRdq5iYGjpGKuhoHY1bdq0UbnDqkwmg1wux+LFi9GuXTuNFkdERESFl/vyKhTjUnIVioLaI0cWL14MPz8/REVFISsrC59//jkuXbqEJ0+e4MSJE1LUSERERAVIePoclxJe3HrjYdqL23EYGpTxsOHh4YELFy7gu+++g6GhITIyMtCrVy8EBQXB0dFRihqJiIgoH7lygW4rj+Pxy4GhCqZGhlqqKH9FuibGwcEBs2fP1nQtJSokJAQhISHIzc3VdilERERF8m92rjJoeDlXgIEMcLAyQ4saFbVcmSq1r0YBgOTkZKxfvx5XrlyBTCZD3bp1MXToUNjaav9mL+ri1ShERFRWpTzPhtfsgwCAf+YGwsSoZAeGSnY1ypEjR+Dm5oZvv/0WycnJePLkCb799lu4ubnhyJEjxSqaiIiICk8xIBQAjErZOI3XqX0aJSgoCH369FGO2QCA3NxcjBkzBkFBQbh48aLGiyQiIqK8cuQvbrhmIAMMSnHYULtn4+bNm5g8ebIyaACAoaEhJk2ahJs3b2q0OCIiIipYjmJeDYPSNa/Gm9Tu2fDx8cGVK1fg7u6u0n7lyhV4e3trqi4iIiJ6TVaOHH/HPsbzrFcXNjxKfzE4tLRd6vqmQoWNCxcuKP9/3LhxGD9+PG7cuIHmzZsDAE6dOoWQkBAsWLBAmiqJiIj0XGjkDaz443q+z5X0wFB1FepqFAMDA8hksnfeRl4mk5WZS0lfv/T1n3/+4dUoRERUqk3Zdh7bzt6Fk7UZ7K3NVJ7r7uWEoS3dSrymwl6NUqiejdjYWI0VVloEBQUhKChI+UYRERGVZoorT4a0dMXHbWpouRr1FCpsFPd27kRERFQ8OS/DhoGsdI/PyE+RZhC9d+8eTpw4gaSkJMhfXnajMG7cOI0URkRERK/kCsWVJ3oQNjZs2IDRo0fDxMQEFStWhOy1hCWTyRg2iIiIJJD78jLX0n7lSX7UDhszZ87EzJkzMW3aNBiU8ut6iYiIdIWiZ8OwDH73qh02nj17hg8//JBBg4iISENuPUxH8rOsty7z5OUN1/TiNMrw4cOxbds2TJ06VYp6iIiI9Mqx6w8xcP3pQi+vF6dR5s+fj65du2L//v3w9PSEsbGxyvPLli3TWHFS4i3miYioNIh9lAEAKGdiiMqWpm9d1q68KVrVsiuJsjRK7bAxb948HDhwQDld+ZsDRMsKzrNBRESlgWL+jPZ1KmNVPx8tVyMNtcPGsmXL8MMPP2DIkCESlENERKRfFGGjLJ4eKSy1R3mampqiZcuWUtRCRESkd+SKq0zK0NkBdakdNsaPH4+VK1dKUQsREZHeyX05N6aBDvdsqH0a5fTp0zh06BD27NmDevXq5RkgumPHDo0VR0REpOv0oWdD7bBRoUIF9OrVS4paiIiI9I5izAZ7Nl6zYcMGKeogIiLSS68GiGq5EAkV6UZsRERE9MqOc3cRcflBkV77z4M0AGXzbq6FpXbYcHNze+t8Grdu3SpWQSWFk3oREZGmBO+6hLR/c4q1Drvyb5/QqyxTO2xMmDBB5XF2djaio6Oxf/9+TJkyRVN1SY6TehERkaZkZr+4pGRKgDuszI3fsXReFiaGCKjnoOmySg21w8b48ePzbQ8JCUFUVFSxCyIiIiprFHdk7d2oKuytzLRcTemjseEogYGB2L59u6ZWR0REVGYoLl/V4WEXxaKxsPHbb7/B1tZWU6sjIiIqE4QQeJk1dHqujOJQ+zRKw4YNVQaICiFw//59PHz4EKGhoRotjoiIqLR7eeUqAN2+oqQ41A4bPXv2VHlsYGCASpUqwc/PD3Xq1NFUXURERGVC7mtpQ5cn5ioOtcNGcHCwFHUQERGVSYrxGgDArJE/HZ6vjIiISHqvhw1dvk18cRS6Z8PAwOCtk3kBgEwmQ05O8SY1ISIiKks4ZuPdCh02du7cWeBzJ0+exMqVKyFeS3dERET64PWeDWaN/BU6bPTo0SNP29WrVzFt2jTs3r0b/fv3x9dff63R4oiIiNRx82E6+n//Nx5nZJbYNl//O5uXvuavSDdiS0hIQHBwMDZu3IiAgADExMSgfv36mq5NUrw3ChGR7jkT+wT3U//VyrYbVLXmmI0CqBU2UlJSMG/ePKxcuRLe3t74888/0bp1a6lqkxTvjUJEpHsU4yda17LD4t5eJbrtSpam7xzbqK8KHTYWLVqEhQsXwsHBAb/++mu+p1WIiIi0STF+opyJIRyseY+S0qLQYWPq1KkwNzdHzZo1sXHjRmzcuDHf5Xbs2KGx4oiIiNShuFCBV4WULoUOG4MGDWL3EBERlWqK0ygMG6VLocNGWFiYhGUQEREVH+++WjpxBlEiItIZ7NkonRg2iIhIZ7was6HlQkgFwwYREekMwZ6NUolhg4iIdMarMRsMG6UJwwYREemMV2M2tFsHqWLYICIincGrUUonhg0iItIZnNSrdGLYICIinaEYIMoxG6VLke76SkREuiPleTYG/3AaCU+fa7uUYsvIzAHAMRuljd6GDd5inojohZj4p4iJf6rtMjSqtr2ltkug18iE4gSXnlLcYj4lJQVWVlbaLoeIqMQdvpqEoWFnULNyeXz7YUNtl1Ns5UwM4Wpnoe0y9EJhv0P1tmeDiIheEHh1W3YPJ/7RRZrHAaJERHpOOahSu2WQDmPYICLSc3JewUESY9ggItJzghNhkcQYNoiI9JziKgFmDZIKwwYRkZ7jrJskNYYNIiI992rWTe3WQbqLYYOISM+9Oo3CtEHSYNggItJzvFMqSY1hg4hIz/E0CkmNYYOISM/xNApJjWGDiEjPKa9G4TcCSYS/WkREeu7VdOXs2SBpMGwQEek5xY3YOGaDpMKwQUSk5+TyF//lvVFIKgwbRER6jtOVk9QYNoiI9BxvxEZSY9ggItJzigGivDcKScVI2wUQke5bc+QmLiWkarsMKsCdJ88A8DQKSUdvw0ZISAhCQkKQm5ur7VKIdNq9p88xf99VbZdBhWBjYaLtEkhHyYTiZJ2eSk1NhbW1NVJSUmBlZaXtcoh0zs2H6eiw9AhMjQzwRec62i6HCmBsZIDO9RxQydJU26VQGVLY71C97dkgopKh+HPGzNgQw1q5abcYItIKDhAlohLBsYdE+othg4gkptdnaokIDBtEJLFX990gIn3FsEFEklLOTsnzKER6i2GDiCTFng0iYtggohLBjg0i/cWwQUSSEhwgSqT3GDaISFKvpg1k1waRvmLYICJJKcdsMGsQ6S2GDSKSlOI0CrMGkf5i2CAiIiJJMWwQkaR4GoWIGDaIqETIeCKFSG8xbBCRpNizQUQMG0RERCQphg0ikhSvRiEihg0iktSr0yiMG0T6imGDiCTFycqJiGGDiCQlBOMGkb5j2CCiEsGzKET6i2GDiCSl6Ndg2CDSXwwbRCQp5QBRXo9CpLcYNohIYi8vfWXWINJbDBtEJCmODyUihg0iKhHs2CDSXwwbRCSpVwNEGTeI9BXDBhFJ6tUAUSLSVwwbRCQpTupFRDoRNvbs2QN3d3fUqlUL69at03Y5RPQaZdRg1waR3jLSdgHFlZOTg0mTJuHw4cOwsrKCj48PevXqBVtbW22XRkSvYdYg0l9lvmfj9OnTqFevHqpUqQJLS0t06dIFBw4c0HZZRPQS7/pKRFoPG0ePHkW3bt3g5OQEmUyG8PDwPMuEhobCzc0NZmZmaNSoEY4dO6Z8LiEhAVWqVFE+rlq1Ku7du1cSpRNRIQje95VI72n9NEpGRga8vLwwdOhQfPDBB3me37JlCyZMmIDQ0FC0bNkSa9asQWBgIC5fvoxq1arlO/hMm39BPUj9F9FxyVrbPlFpc/V+GgCeRiHSZ1oPG4GBgQgMDCzw+WXLlmH48OEYMWIEAGDFihU4cOAAvvvuO8yfPx9VqlRR6cm4e/cumjVrVuD6MjMzkZmZqXycmpqqgb14JTruKUZvOqfRdRLpAiNDrXekEpGWaD1svE1WVhbOnj2LqVOnqrT7+/vj5MmTAICmTZvi4sWLuHfvHqysrLB3717MnDmzwHXOnz8fs2fPlqzmCuWM0djFRrL1E5VFBjIZBrZw0XYZRKQlpTpsPHr0CLm5ubC3t1dpt7e3x/379wEARkZGWLp0Kdq1awe5XI7PP/8cFStWLHCd06ZNw6RJk5SPU1NT4ezsrLGam1eviN8+8dXY+oiIiMq6Uh02FN4cgyGEUGnr3r07unfvXqh1mZqawtTUVKP1ERERUcFK9UlUOzs7GBoaKnsxFJKSkvL0dhAREVHpVKrDhomJCRo1aoSIiAiV9oiICPj68lQFERFRWaD10yjp6em4ceOG8nFsbCxiYmJga2uLatWqYdKkSRg4cCAaN26MFi1aYO3atYiLi8Po0aOLtd2QkBCEhIQgNze3uLtAREREbyETWr5LUmRkJNq1a5enffDgwQgLCwPwYlKvRYsWITExEfXr18fy5cvRpk0bjWw/NTUV1tbWSElJgZWVlUbWSUREpA8K+x2q9bChbQwbRERERVPY79BSPWaDiIiIyj6GDSIiIpIUwwYRERFJSm/DRkhICDw8PNCkSRNtl0JERKTTOECUA0SJiIiKpLDfoVqfZ0PbFFlL03d/JSIi0nWK78539VvofdhIS0sDAI3ejI2IiEifpKWlwdrausDn9f40ilwuR0JCAiwtLfPc8K2oFHeSjY+P56kZLeNnUXrwsyhd+HmUHmX5sxBCIC0tDU5OTjAwKHgYqN73bBgYGKBq1aqSrNvKyqrM/eLoKn4WpQc/i9KFn0fpUVY/i7f1aCjo7dUoREREVDIYNoiIiEhSDBsSMDU1RXBwMExNTbVdit7jZ1F68LMoXfh5lB768Fno/QBRIiIikhZ7NoiIiEhSDBtEREQkKYYNIiIikhTDBhEREUmKYaOIQkND4ebmBjMzMzRq1AjHjh176/JHjhxBo0aNYGZmhurVq2P16tUlVKnuU+eziIyMhEwmy/Nz9erVEqxYNx09ehTdunWDk5MTZDIZwsPD3/kaHhfSUPez4HEhnfnz56NJkyawtLRE5cqV0bNnT1y7du2dr9O1Y4Nhowi2bNmCCRMmYPr06YiOjkbr1q0RGBiIuLi4fJePjY1Fly5d0Lp1a0RHR+PLL7/EuHHjsH379hKuXPeo+1koXLt2DYmJicqfWrVqlVDFuisjIwNeXl5YtWpVoZbncSEddT8LBR4XmnfkyBEEBQXh1KlTiIiIQE5ODvz9/ZGRkVHga3Ty2BCktqZNm4rRo0ertNWpU0dMnTo13+U///xzUadOHZW2UaNGiebNm0tWo75Q97M4fPiwACCSk5NLoDr9BUDs3LnzrcvwuCgZhfkseFyUnKSkJAFAHDlypMBldPHYYM+GmrKysnD27Fn4+/urtPv7++PkyZP5vuavv/7Ks3xAQACioqKQnZ0tWa26riifhULDhg3h6OiIDh064PDhw1KWSQXgcVH68LiQXkpKCgDA1ta2wGV08dhg2FDTo0ePkJubC3t7e5V2e3t73L9/P9/X3L9/P9/lc3Jy8OjRI8lq1XVF+SwcHR2xdu1abN++HTt27IC7uzs6dOiAo0ePlkTJ9BoeF6UHj4uSIYTApEmT0KpVK9SvX7/A5XTx2ND7u74W1Zu3oxdCvPUW9fktn187qU+dz8Ld3R3u7u7Kxy1atEB8fDyWLFmCNm3aSFon5cXjonTgcVEyxo4diwsXLuD48ePvXFbXjg32bKjJzs4OhoaGef5yTkpKypNEFRwcHPJd3sjICBUrVpSsVl1XlM8iP82bN8f169c1XR69A4+L0o3HhWZ9+umn2LVrFw4fPoyqVau+dVldPDYYNtRkYmKCRo0aISIiQqU9IiICvr6++b6mRYsWeZY/ePAgGjduDGNjY8lq1XVF+SzyEx0dDUdHR02XR+/A46J043GhGUIIjB07Fjt27MChQ4fg5ub2ztfo5LGhxcGpZdbmzZuFsbGxWL9+vbh8+bKYMGGCsLCwELdv3xZCCDF16lQxcOBA5fK3bt0S5cqVExMnThSXL18W69evF8bGxuK3337T1i7oDHU/i+XLl4udO3eKf/75R1y8eFFMnTpVABDbt2/X1i7ojLS0NBEdHS2io6MFALFs2TIRHR0t7ty5I4TgcVGS1P0seFxI55NPPhHW1tYiMjJSJCYmKn+ePXumXEYfjg2GjSIKCQkRLi4uwsTERPj4+KhcxjR48GDRtm1bleUjIyNFw4YNhYmJiXB1dRXfffddCVesu9T5LBYuXChq1KghzMzMhI2NjWjVqpX473//q4WqdY/i8sk3fwYPHiyE4HFRktT9LHhcSCe/zwGA2LBhg3IZfTg2eIt5IiIikhTHbBAREZGkGDaIiIhIUgwbREREJCmGDSIiIpIUwwYRERFJimGDiIiIJMWwQURERJJi2CAiItJRR48eRbdu3eDk5ASZTIbw8HC11yGEwJIlS1C7dm2YmprC2dkZ8+bNU2sdDBtEVCi3b9+GTCZDTEyMtktRunr1Kpo3bw4zMzN4e3truxyiUicjIwNeXl5YtWpVkdcxfvx4rFu3DkuWLMHVq1exe/duNG3aVK11MGwQlRFDhgyBTCbDggULVNrDw8PL7G2niys4OBgWFha4du0a/vzzT22XQ1TqBAYGYu7cuejVq1e+z2dlZeHzzz9HlSpVYGFhgWbNmiEyMlL5/JUrV/Ddd9/h999/R/fu3eHm5gZvb2907NhRrToYNojKEDMzMyxcuBDJycnaLkVjsrKyivzamzdvolWrVnBxcSmTt94uzr4TacLQoUNx4sQJbN68GRcuXMB//vMfdO7cGdevXwcA7N69G9WrV8eePXvg5uYGV1dXjBgxAk+ePFFrOwwbRGVIx44d4eDggPnz5xe4zKxZs/KcUlixYgVcXV2Vj4cMGYKePXti3rx5sLe3R4UKFTB79mzk5ORgypQpsLW1RdWqVfHDDz/kWf/Vq1fh6+sLMzMz1KtXT+WvIAC4fPkyunTpgvLly8Pe3h4DBw7Eo0ePlM/7+flh7NixmDRpEuzs7NCpU6d890Mul2POnDmoWrUqTE1N4e3tjf379yufl8lkOHv2LObMmQOZTIZZs2blux7F9saOHYsKFSqgYsWK+Oqrr/D6baE2bdqExo0bw9LSEg4ODujXrx+SkpKUz0dGRkImk+G///0vvLy8YGZmhmbNmuF///ufyrZOnjyJNm3awNzcHM7Ozhg3bhwyMjKUz7u6umLu3LkYMmQIrK2tMXLkSGRlZWHs2LFwdHSEmZkZXF1d3/r5EmnKzZs38euvv2Lbtm1o3bo1atSogc8++wytWrXChg0bAAC3bt3CnTt3sG3bNvz4448ICwvD2bNn0bt3b7W2xbBBVIYYGhpi3rx5WLlyJe7evVusdR06dAgJCQk4evQoli1bhlmzZqFr166wsbHB33//jdGjR2P06NGIj49Xed2UKVMwefJkREdHw9fXF927d8fjx48BAImJiWjbti28vb0RFRWF/fv348GDB+jTp4/KOjZu3AgjIyOcOHECa9asybe+//u//8PSpUuxZMkSXLhwAQEBAejevbvyL67ExETUq1cPkydPRmJiIj777LMC91Wxvb///hvffvstli9fjnXr1imfz8rKwtdff43z588jPDwcsbGxGDJkSJ71TJkyBUuWLMGZM2dQuXJldO/eHdnZ2QCA//3vfwgICECvXr1w4cIFbNmyBcePH8fYsWNV1rF48WLUr18fZ8+exYwZM/Dtt99i165d2Lp1K65du4ZNmzapBEMiqZw7dw5CCNSuXRvly5dX/hw5cgQ3b94E8CL0Z2Zm4scff0Tr1q3h5+eH9evX4/Dhw7h27VrhN6bVe84SUaENHjxY9OjRQwghRPPmzcWwYcOEEELs3LlTvH4oBwcHCy8vL5XXLl++XLi4uKisy8XFReTm5irb3N3dRevWrZWPc3JyhIWFhfj111+FEELExsYKAGLBggXKZbKzs0XVqlXFwoULhRBCzJgxQ/j7+6tsOz4+XgAQ165dE0II0bZtW+Ht7f3O/XVychLffPONSluTJk3EmDFjlI+9vLxEcHDwW9fTtm1bUbduXSGXy5VtX3zxhahbt26Brzl9+rQAINLS0oQQr27ZvnnzZuUyjx8/Fubm5mLLli1CCCEGDhwoPv74Y5X1HDt2TBgYGIjnz58LIYRwcXERPXv2VFnm008/Fe3bt1epj0gKAMTOnTuVjzdv3iwMDQ3F1atXxfXr11V+EhMThRBCzJw5UxgZGams59mzZwKAOHjwYKG3zZ4NojJo4cKF2LhxIy5fvlzkddSrVw8GBq/+CbC3t4enp6fysaGhISpWrKhyOgEAWrRoofx/IyMjNG7cGFeuXAEAnD17FocPH1b5K6lOnToAoPxLCQAaN2781tpSU1ORkJCAli1bqrS3bNlSuS11NG/eXGUQbYsWLXD9+nXk5uYCAKKjo9GjRw+4uLjA0tISfn5+AIC4uDiV9by+77a2tnB3d1fZ97CwMJV9DwgIgFwuR2xsbIH7PmTIEMTExMDd3R3jxo3DwYMH1d4/oqJo2LAhcnNzkZSUhJo1a6r8ODg4AHhxzOXk5Kgcv//88w8AwMXFpdDbMtJs6URUEtq0aYOAgAB8+eWXebr7DQwMVMYjAFB29b/O2NhY5bFMJsu3TS6Xv7MexRe5XC5Ht27dsHDhwjzLODo6Kv/fwsLinet8fb0KQgiNX3mTkZEBf39/+Pv7Y9OmTahUqRLi4uIQEBBQqAGcr+/7qFGjMG7cuDzLVKtWTfn/b+67j48PYmNjsW/fPvzxxx/o06cPOnbsiN9++62Ye0YEpKen48aNG8rHsbGxiImJga2tLWrXro3+/ftj0KBBWLp0KRo2bIhHjx7h0KFD8PT0RJcuXdCxY0f4+Phg2LBhWLFiBeRyOYKCgtCpUyfUrl270HUwbBCVUQsWLIC3t3eeA75SpUq4f/++yhezJufGOHXqFNq0aQMAyMnJwdmzZ5XjEnx8fLB9+3a4urrCyKjo/7xYWVnByckJx48fV24LeDEAU93r+xU1v/m4Vq1aMDQ0xNWrV/Ho0SMsWLAAzs7OAICoqKgC16MIDsnJyfjnn3+UPTc+Pj64dOkSatasqXZ9VlZW6Nu3L/r27YvevXujc+fOePLkCWxtbdVeF9HroqKi0K5dO+XjSZMmAQAGDx6MsLAwbNiwAXPnzsXkyZNx7949VKxYES1atECXLl0AvPjjZffu3fj000/Rpk0bWFhYIDAwEEuXLlWrDoYNojLK09MT/fv3x8qVK1Xa/fz88PDhQyxatAi9e/fG/v37sW/fPlhZWWlkuyEhIahVqxbq1q2L5cuXIzk5GcOGDQMABAUF4fvvv8dHH32EKVOmwM7ODjdu3MDmzZvx/fffw9DQsNDbmTJlCoKDg1GjRg14e3tjw4YNiImJwc8//6x2zfHx8Zg0aRJGjRqFc+fOYeXKlcp/LKtVqwYTExOsXLkSo0ePxsWLF/H111/nu545c+agYsWKsLe3x/Tp02FnZ4eePXsCAL744gs0b94cQUFBGDlyJCwsLHDlyhVERETk+Yxet3z5cjg6OsLb2xsGBgbYtm0bHBwcUKFCBbX3k+hNfn5+eXo6X2dsbIzZs2dj9uzZBS7j5OSE7du3F6sOjtkgKsO+/vrrPP+Q1K1bF6GhoQgJCYGXlxdOnz791is11LVgwQIsXLgQXl5eOHbsGH7//XfY2dkBePGP0okTJ5Cbm4uAgADUr18f48ePh7W1tcr4kMIYN24cJk+ejMmTJ8PT0xP79+/Hrl27UKtWLbVrHjRoEJ4/f46mTZsiKCgIn376KT7++GMAL3qCwsLCsG3bNnh4eGDBggVYsmRJgfs+fvx4NGrUCImJidi1axdMTEwAAA0aNMCRI0dw/fp1tG7dGg0bNsSMGTNUTh/lp3z58li4cCEaN26MJk2a4Pbt29i7d6/a7xdRaSYTb4s8RERlnJ+fH7y9vbFixYoiryMyMhLt2rVDcnIyexyIioDRmYiIiCTFsEFERESS4mkUIiIikhR7NoiIiEhSDBtEREQkKYYNIiIikhTDBhEREUmKYYOIiIgkxbBBREREkmLYICIiIkkxbBAREZGk/h+YamTpBV2xMAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "AI_S2 = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/S2_Kwds_AI.parquet\")\n",
    "counts = sorted([el[0] for el in AI_S2.select(\"Kwd_count\").collect()])\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.figure(figsize=(6,4))\n",
    "plt.semilogy(np.linspace(0,AI_S2.count(), len(counts)), counts)\n",
    "plt.xlabel(\"Number of papers\")\n",
    "plt.ylabel(\"Number of keywords\")\n",
    "plt.title(\"Cumulative number of papers in S2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "df99b045-3d05-485c-9641-5252abe1b3de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAGHCAYAAAAZa5AYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABHTklEQVR4nO3deXwM9/8H8Nfmvi9CxBHqiDuROEMkQaRRV1VplbpLxa2UKkFb9/WrRlWpaFFH3apISSjaEhK+jqhbSCKOSIgiyX5+f+iOrE3IJjvZTfb1fDzyYD47O/OemUz2vZ9rFEIIASIiIiKZmOg7ACIiIirdmGwQERGRrJhsEBERkayYbBAREZGsmGwQERGRrJhsEBERkayYbBAREZGsmGwQERGRrJhsEBERkayYbBiB06dPo3///qhWrRqsrKxgZ2cHHx8fzJ07F/fv39d3eK80bdo0KBSKQr139+7dmDZtWp6vVa1aFf369St8YKVIv379YGdnp+8wCmzDhg2oV68erK2toVAoEB8fr++QDE6/fv1QtWpVnW83MTERw4cPR/Xq1WFlZQVnZ2cEBgZi7dq1MOTJqH18fKBQKDB//vw8X4+MjIRCoUBsbKxUpvrbo/oxNzdHlSpVMHjwYKSkpGhso2rVqmrr5/4JDAyU69BKDDN9B0Dy+v777zFs2DB4enpi/PjxqFu3LrKyshAbG4tly5bhzz//xNatW/Udpix2796NiIiIPBOOrVu3wsHBofiDoiK5c+cO+vTpgzfffBNLly6FpaUlatWqpe+wDM6UKVMwatQonW7zyJEj6NixI+zs7DB+/Hg0bNgQ6enp2LhxI3r37o2dO3di3bp1MDExrO+w8fHxiIuLAwCsXLkSn3zyiVbv37NnDxwdHfHo0SPs27cPCxYswNGjRxEfHw9zc3O1dVu2bJlnQsO/NUw2SrU///wTH3/8MYKDg7Ft2zZYWlpKrwUHB2PcuHHYs2ePHiPUn0aNGuk7BKPz+PFj2NjYFGkb//zzD7KystC7d28EBAToKDLDU9RzVb16dR1GAzx48ADdunWDo6Mj/v77b5QvX156rUuXLmjYsCEmTpwIb29vTJw4Uaf7LqoVK1YAAN566y38+uuvOHr0KPz8/Ar8fl9fX5QtWxYA0K5dO9y9exerVq3C4cOHERQUpLauk5MTmjdvrrvgSxHDSkFJp2bOnAmFQoHly5erJRoqFhYW6Ny5s7SsUCjyrAV4uclBVeV44MABDB48GGXKlIGDgwM+/PBDZGZmIiUlBT169ICTkxMqVKiATz75BFlZWdL7Y2JioFAoEBMTo7afa9euQaFQIDIy8pXHtWHDBrRv3x4VKlSAtbU16tSpg4kTJyIzM1Nap1+/foiIiJCOS/Vz7do1jWO6c+cOLCwsMGXKFI19JSQkQKFQ4Ouvv5bKUlJSMGTIEFSqVAkWFhaoVq0apk+fjuzs7FfGrdpvx44dsWfPHvj4+MDa2hq1a9fGDz/8oLZefs1HqnOvOo7c29y1axcaNWoknZNdu3ZJ76lTpw5sbW3RtGlTtari3M6ePYu2bdvC1tYWrq6uGD58OB4/fqy2jhACS5cuhbe3N6ytreHs7Izu3bvjypUrausFBgaifv36OHToEPz8/GBjY4MBAwa88tzs2LEDLVq0gI2NDezt7REcHIw///xTer1fv35o1aoVAKBnz56vrZ5WnauoqCj0798fLi4usLW1RadOnTTijYqKQpcuXVCpUiVYWVmhRo0aGDJkCO7evau2nuq6xMXFoVu3bnBwcICjoyN69+6NO3fuaMSwYcMGtGjRAra2trCzs0NISIj0LTv3cdnZ2eF///sf2rdvD3t7e7Rt2xYAEBcXh44dO6JcuXKwtLSEu7s73nrrLdy8efOV5zKvZhSFQoHhw4fjp59+Qp06dWBjYwMvLy/p9+RVVqxYgdTUVMyePVst0VCZMGECateujXnz5kn3uuo+X7NmDcaOHQs3NzdYW1sjICBA4xwAQGxsLDp37gwXFxdYWVmhUaNG2Lhxo9o6qmsaHR2Njz/+GGXLlkWZMmXQrVs3JCUlaWzzyZMnWLduHXx9fbFo0SIA0LjXtNW4cWMAwO3bt4u0HWPDZKOUysnJwYEDB+Dr64vKlSvLso9BgwbB0dER69evx+eff45169Zh8ODBeOutt+Dl5YVffvkFffv2xYIFC7BkyRKd7ffixYvo0KEDVq5ciT179mD06NHYuHEjOnXqJK0zZcoUdO/eHcDzGh7VT4UKFTS25+rqio4dO2L16tVQKpVqr61atQoWFhb44IMPADxPNJo2bYq9e/di6tSp+O233zBw4EDMmjULgwcPLlD8p06dwrhx4zBmzBhs374dDRs2xMCBA3Ho0KHCnhKcOnUKkyZNwqeffootW7bA0dER3bp1Q3h4OFasWIGZM2di7dq1SE9PR8eOHfHvv/+qvT8rKwsdOnRA27ZtsW3bNgwfPhzfffcdevbsqbbekCFDMHr0aLRr1w7btm3D0qVLcfbsWfj5+Wn88U1OTkbv3r3Rq1cv7N69G8OGDcs3/nXr1qFLly5wcHDAzz//jJUrVyItLQ2BgYE4fPgwgOfXVJVAzpw5E3/++SeWLl362nMzcOBAmJiYYN26dVi8eDGOHTuGwMBAPHjwQFrn8uXLaNGiBb799lvs27cPU6dOxd9//41WrVqpJcoqb7/9NmrUqIFffvkF06ZNw7Zt2xASEqK27syZM/H++++jbt262LhxI3766Sc8fPgQ/v7+OHfunNr2nj17hs6dO6NNmzbYvn07pk+fjszMTAQHB+P27duIiIhAVFQUFi9ejCpVquDhw4evPe68/Prrr/jmm28wY8YMbN68GS4uLnj77bc1kq+XRUVFwdTUVO0ey02hUKBz5864f/8+Tpw4ofbaZ599hitXrmDFihVYsWIFkpKSEBgYqLbP6OhotGzZEg8ePMCyZcuwfft2eHt7o2fPnnl++Rg0aBDMzc2xbt06zJ07FzExMejdu7fGelu2bEFaWhoGDBiAmjVrolWrVtiwYQMePXpUgLOVt6tXrwJAns13QghkZ2dr/Bhyf5ZiI6hUSklJEQDEe++9V+D3ABDh4eEa5R4eHqJv377S8qpVqwQAMWLECLX1unbtKgCIhQsXqpV7e3sLHx8faTk6OloAENHR0WrrXb16VQAQq1atksrCw8PFq35NlUqlyMrKEgcPHhQAxKlTp6TXwsLC8n3vy8e0Y8cOAUDs27dPKsvOzhbu7u7inXfekcqGDBki7OzsxPXr19W2N3/+fAFAnD17Nt9YVfu1srJSe/+///4rXFxcxJAhQ1573Kpzf/XqVbVtWltbi5s3b0pl8fHxAoCoUKGCyMzMlMq3bdsmAIgdO3ZIZX379hUAxP/93/+p7eurr74SAMThw4eFEEL8+eefAoBYsGCB2nqJiYnC2tpaTJgwQSoLCAgQAMT+/ftfeT6EECInJ0e4u7uLBg0aiJycHKn84cOHoly5csLPz08qU/3ubNq06bXbVZ2rt99+W638yJEjAoD48ssv83yf6nfq+vXrAoDYvn279JrquowZM0btPWvXrhUAxJo1a4QQQty4cUOYmZlp3CMPHz4Ubm5uokePHlKZ6vz/8MMPauvGxsYKAGLbtm2vPdaX9e3bV3h4eKiVARDly5cXGRkZUllKSoowMTERs2bNeuX2ateuLdzc3F65zrfffisAiA0bNgghXlwrHx8foVQqpfWuXbsmzM3NxaBBg9S236hRI5GVlaW2zY4dO4oKFSpIvxeqazps2DC19ebOnSsAiOTkZLXyNm3aCCsrK5GWlqb2/pUrV6qtpyo/fvy4VKa61ikpKSIrK0ukpaWJjRs3CltbW/H+++9rHL+Hh4cAkOfPF1988cpzZwxYs0GF1rFjR7XlOnXqAHjeNvpy+fXr13W23ytXrqBXr15wc3ODqakpzM3Npfb78+fPF2qboaGhcHNzw6pVq6SyvXv3IikpSa36f9euXQgKCoK7u7vaN5fQ0FAAwMGDB1+7L29vb1SpUkVatrKyQq1atYp0jry9vVGxYkVpWXUtAgMD1dr+VeV57UtVe6PSq1cvAM+/dQLPj12hUKB3795qx+7m5gYvLy+NZjFnZ2e0adPmtbFfuHABSUlJ6NOnj1rnQjs7O7zzzjv466+/NJpztPHycfn5+cHDw0M6LgBITU3F0KFDUblyZZiZmcHc3BweHh4A8v6denmbPXr0gJmZmbTNvXv3Ijs7Gx9++KHaubKyskJAQIDGuQKAd955R225Ro0acHZ2xqeffoply5Zp1IYURlBQEOzt7aXl8uXLo1y5cjq5P8V/395fbv7r1auXWpmHhwf8/Pykc3Xp0iUkJCRI5zT3+erQoQOSk5Nx4cIFtW3mbv4FgIYNGwJQ/72+evUqoqOj0a1bNzg5OQEA3n33Xdjb22vVlOLm5gZzc3M4OzujR48e8PX1xerVq/Nct1WrVjh+/LjGz8CBAwu8v9KKHURLqbJly8LGxkaq8pODi4uL2rKFhUW+5U+ePNHJPh89egR/f39YWVnhyy+/RK1atWBjY4PExER069ZNo3mgoMzMzNCnTx8sWbIEDx48gJOTEyIjI1GhQgWEhIRI692+fRs7d+7U6IWu8nIbf17KlCmjUWZpaVno2AHtrgUAjethZmamEZebmxsA4N69ewCeH7sQIs82ewB444031JbzarLKi2r7ea3v7u4OpVKJtLS0QneYVB3Hy2Wq/SqVSrRv3x5JSUmYMmUKGjRoAFtbWyiVSjRv3jzP6/LyNlXnL/e5AoAmTZrkGdPLIzZsbGw0Riw4Ojri4MGD+Oqrr/DZZ58hLS0NFSpUwODBg/H555/n+zv4KoX93atSpQouXryIzMxM2Nra5rmOqh/Ry822+Z3/U6dOAXhxrj755JN8R4q8fF+9fByqPmm5j+OHH36AEALdu3dXazLr3Lkz1q5di4SEBNSuXTvP/eX2+++/w9HREffv38fy5cuxefNmjBgxAsuWLdNY19HRUerTQeqYbJRSpqamaNu2LX777TfcvHkTlSpVeu17LC0t8fTpU41y1R9QXbGysgIAjX0V5IP6wIEDSEpKQkxMjNpohNx/TAqrf//+mDdvHtavX4+ePXtix44dGD16NExNTaV1ypYti4YNG+Krr77Kcxvu7u5FjgNQP0e5O/cW5BwVRnZ2Nu7du6f2R1w1l4CqrGzZslAoFPjjjz/y7HD8cllB50dRbT85OVnjtaSkJJiYmMDZ2blgB5KHvOZESElJQY0aNQAAZ86cwalTpxAZGYm+fftK61y6dOmV28xdk/Ty+VONXvjll1+kGpJXye9cNWjQAOvXr4cQAqdPn0ZkZCRmzJgBa2vrYh31ERwcjH379mHnzp147733NF4XQmDHjh1wcXGBr6+v2mv5nf+Xz9WkSZPQrVu3PPfv6empVbxKpVLq65HfNn/44QfMnTv3tdvy8vKSYgwODkZISAiWL1+OgQMH5ptMkiY2o5RikyZNghACgwcPxrNnzzRez8rKws6dO6XlqlWr4vTp02rrHDhwoEidqfKi6iX/8r527Njx2veq/ii//MH23Xffaayb17edV6lTpw6aNWuGVatWYd26dXj69Cn69++vtk7Hjh1x5swZVK9eHY0bN9b40VWykd85yn29dG3t2rVqy+vWrQMAacRHx44dIYTArVu38jz2Bg0aFGq/np6eqFixItatW6fWkS4zMxObN2+WRqgU1svHdfToUVy/fl06Lm1+p/Lb5saNG5GdnS1tMyQkBGZmZrh8+XKe50rbb78KhQJeXl5YtGgRnJyccPLkSa3eX1SDBg1CuXLlMGnSJKSmpmq8PnfuXCQkJGDChAkaNS4///yz2nW9fv06jh49Kp0rT09P1KxZE6dOncr3XOVu+imIvXv34ubNmwgLC0N0dLTGT7169fDjjz8WaARZbgqFAhERETA1NcXnn3+u1XuNHWs2SjFV7/phw4bB19cXH3/8MerVq4esrCzExcVh+fLlqF+/vtTDvE+fPpgyZQqmTp2KgIAAnDt3Dt988w0cHR11GpebmxvatWuHWbNmwdnZGR4eHti/fz+2bNny2vf6+fnB2dkZQ4cORXh4OMzNzbF27VqpSjY31YffnDlzEBoaClNTUzRs2FBqSsjLgAEDMGTIECQlJcHPz0/jG9WMGTMQFRUFPz8/jBw5Ep6ennjy5AmuXbuG3bt3Y9myZQWqRXqdDh06wMXFBQMHDsSMGTNgZmaGyMhIJCYmFnnbebGwsMCCBQvw6NEjNGnSBEePHsWXX36J0NBQabhpy5Yt8dFHH6F///6IjY1F69atYWtri+TkZBw+fBgNGjTAxx9/rPW+TUxMMHfuXHzwwQfo2LEjhgwZgqdPn2LevHl48OABZs+eXaRji42NxaBBg/Duu+8iMTERkydPRsWKFaXRMbVr10b16tUxceJECCHg4uKCnTt3IioqKt9tbtmyBWZmZggODsbZs2cxZcoUeHl5oUePHgCeJ4szZszA5MmTceXKFbz55ptwdnbG7du3cezYMdja2mL69OmvjHvXrl1YunQpunbtijfeeANCCGzZsgUPHjxAcHBwkc6JtpycnLBlyxZ07NgRvr6+GD9+PLy8vJCRkYENGzZg7dq16NmzJ8aPH6/x3tTUVLz99tsYPHgw0tPTER4eDisrK0yaNEla57vvvkNoaChCQkLQr18/VKxYEffv38f58+dx8uRJbNq0Sat4V65cCTMzM3z22Wd5fgEYMmQIRo4ciV9//RVdunTRats1a9bERx99hKVLl+Lw4cPS/QE8r2H966+/NN5jaWnJuX301DGVilF8fLzo27evqFKlirCwsBC2traiUaNGYurUqSI1NVVa7+nTp2LChAmicuXKwtraWgQEBIj4+Ph8R6Pk7rktxIve23fu3FEr79u3r7C1tVUrS05OFt27dxcuLi7C0dFR9O7dW+p9/7rRKEePHhUtWrQQNjY2wtXVVQwaNEicPHlS471Pnz4VgwYNEq6urkKhUKiN4nj5mFTS09OFtbW1ACC+//77PM/nnTt3xMiRI0W1atWEubm5cHFxEb6+vmLy5Mni0aNHeb5HxcPDQ7z11lsa5QEBASIgIECt7NixY8LPz0/Y2tqKihUrivDwcLFixYo8R6PktU0AIiwsTK1MNeJn3rx5Upnq+pw+fVoEBgYKa2tr4eLiIj7++OM8j+eHH34QzZo1E7a2tsLa2lpUr15dfPjhhyI2NlbteOrVq/fKc/Gybdu2iWbNmgkrKytha2sr2rZtK44cOaK2TmFGo+zbt0/06dNHODk5CWtra9GhQwdx8eJFtXXPnTsngoODhb29vXB2dhbvvvuuuHHjhsYILdXv44kTJ0SnTp2EnZ2dsLe3F++//764fft2nscUFBQkHBwchKWlpfDw8BDdu3cXv//+u7ROXveHEEIkJCSI999/X1SvXl1YW1sLR0dH0bRpUxEZGfnaY89vNMrLvw9C5H8v5OXGjRsiLCxMvPHGG8LCwkI4OjqK1q1bizVr1qiNOBHixbX66aefxMiRI4Wrq6uwtLQU/v7+ar8rKqdOnRI9evQQ5cqVE+bm5sLNzU20adNGLFu2TFonv789uUe43blzR1hYWIiuXbvmexxpaWnC2tpadOrUKd/t5vf3TAghbt++Lezs7ERQUJBU9qrRKBUrVnzNmS39FEJwADARlT6RkZHo378/jh8/rrNOe9OmTcP06dNx584dqR2f8hYTE4OgoCBs2rRJmvOGjBf7bBAREZGsmGwQERGRrNiMQkRERLJizQYRERHJiskGERERyYrJBhEREcnK6Cf1UiqVSEpKgr29fYGnVyYiIqLnU9U/fPgQ7u7uGs/8yc3ok42kpCSNBwcRERFRwSUmJr5y9mSjTTYiIiIQEREhzY2fmJio8dRFIiIiyl9GRgYqV6782ufXGP3Q14yMDDg6OiI9PZ3JBhERkRYK+hnKDqJEREQkKyYbREREJCsmG0RERCQrJhtEREQkKyYbREREJCsmG0RERCQrJhtEREQkK6NNNiIiIlC3bl00adJE36EQERGVapzUi5N6ERERFUpBP0ONdrpyIiIiY3HmVjpupj1GjXJ2qFHu1VOLy8Fom1GIiIiMxc/HbmDompP49XSKXvbPZIOIiIhkxWSDiIiIZMVkg4iIqJTT90gQJhtERERGQqHQz36ZbBAREZGsmGwQERGRrIw22eAMokREZCz0PX2n0SYbYWFhOHfuHI4fP67vUIiIiIqFnrpsGG+yQURERMWDyQYREVGpp992FCYbREREJCsmG0REREaC82wQERFRqcRkg4iIqJTj0FciIiIq1ZhsEBERGQmFnjptMNkgIiIiWTHZICIiKuXYZ0NP+GwUIiKi4mG0yQafjUJERFQ8jDbZICIiMhaC05UTERFRacZkg4iIyEhwunIiIiIqlZhsEBERlXIc+kpERESlGpMNIiIiI6EApysnIiKiUojJBhERUSmn5y4bTDaIiIhIXkw2iIiIjATn2SAiIiJZcOgrERERlWpGm2zwEfNERETFw2iTDT5inoiIjI2eumwYb7JBRERkLPiIeSIiIirVmGwQERGRrJhsEBERGQnOs0FERETy4DwbREREVJox2SAiIiJZMdkgIiIq5VStKAo9zbTBZIOIiIhkxWSDiIiIZMVkg4iIyEhw6CsRERHJQuj5GfNMNoiIiEhWTDaIiIhIVkw2iIiISFZMNoiIiEo5Pc9WzmSDiIiI5GW0yUZERATq1q2LJk2a6DsUIiKiUs1ok42wsDCcO3cOx48f13coREREslKNfFXoaaINo002iIiIqHgw2SAiIiJZMdkgIiIiWTHZICIiKuVePGJeP5hsEBERkayYbBAREZGsmGwQERGRrJhsEBERlXKqR8zraZoNJhtEREQkLyYbREREJCsmG0RERKUcn/pKRERExYLzbBAREVGpxGSDiIiIZMVkg4iIqLQrDY+Yf/DggS42Q0RERKWQ1snGnDlzsGHDBmm5R48eKFOmDCpWrIhTp07pNDgiIiIq+bRONr777jtUrlwZABAVFYWoqCj89ttvCA0Nxfjx43UeIBEREZVsZtq+ITk5WUo2du3ahR49eqB9+/aoWrUqmjVrpvMAiYiIqGgESth05c7OzkhMTAQA7NmzB+3atQPwfN71nJwc3UZHREREJZ7WNRvdunVDr169ULNmTdy7dw+hoaEAgPj4eNSoUUPnARIREVHJpnWysWjRIlStWhWJiYmYO3cu7OzsADxvXhk2bJjOAyQiIqKiEXqer1zrZMPc3ByffPKJRvno0aN1EQ8RERHJRF/TlRco2dixY0eBN9i5c+dCB0NERESlT4GSja5du6otKxQKiFx1MrlnJGMnUSIiIsOSnP5Er/sv0GgUpVIp/ezbtw/e3t747bff8ODBA6Snp2P37t3w8fHBnj175I6XiIiItGRh+vzjPuNJtl72r3WfjdGjR2PZsmVo1aqVVBYSEgIbGxt89NFHOH/+vE4DlEtERAQiIiJYE0NERKWeo405AMDSTD+PRNN6r5cvX4ajo6NGuaOjI65du6aLmIpFWFgYzp07h+PHj+s7FCIiIlkplc+7PthZal3HoBNaJxtNmjTB6NGjkZycLJWlpKRg3LhxaNq0qU6DIyIioqLL+a+fpYlJCXnq68qVK5GamgoPDw/UqFEDNWrUQJUqVZCcnIyVK1fKESMREREVwX8VGzDV03zlWten1KxZE6dOncLvv/+OhIQECCFQt25dtGvXTm1UChERERkGVTOKiX66bGiXbGRnZ8PKygrx8fFo37492rdvL1dcREREpCNKVTOKnioFtMpxzMzM4OHhwREcREREJUjOfzUbpiWlz8bnn3+OSZMm4f79+3LEQ0RERDqmmodTXzUbWvfZ+Prrr3Hp0iW4u7vDw8MDtra2aq+fPHlSZ8ERERFR0eXouRlF62Tj5anLiYiIyLCdvvkAgP6aUbRONsLDw+WIg4iIiGRS1s4SyelP8DRbP30uCz2V2IkTJ3D+/HkoFArUrVsXjRo10mVcREREpCMW/01T7uZgpZf9a51spKam4r333kNMTAycnJwghEB6ejqCgoKwfv16uLq6yhEnERERFVKJG40yYsQIZGRk4OzZs7h//z7S0tJw5swZZGRkYOTIkXLESEREREWg1HOyoXXNxp49e/D777+jTp06UlndunURERHBSb6IiIgMULayBE3qBQBKpRLm5uYa5ebm5lAqlToJioiIiHRHNYOomWkJSTbatGmDUaNGISkpSSq7desWxowZg7Zt2+o0OCIiIio6qc9GSanZ+Oabb/Dw4UNUrVoV1atXR40aNVCtWjU8fPgQS5YskSNGIiIiKoK0x1kASlCfjcqVK+PkyZOIiorSeOorERERGS59PZ1d62Tj8ePHsLGxQXBwMIKDg+WIiYiIiHRENRIFAOytCj29VpFovVcnJyc0btwYgYGBCAwMRMuWLTWej0JERESGISvX4A3V5F7FTeu9Hjx4EJ07d8bJkyfRvXt3ODs7o3nz5pg4cSJ+++03OWIkIiKiQsrKeVGzYWGqn2RDIYQQr18tbzk5OTh+/DiWLVuGtWvXQqlUIidHP/OuF1ZGRgYcHR2Rnp4OBwcHfYdDRESkU/cePYXvl78DAC7P7KDTTqIF/QwtVONNQkICYmJicPDgQcTExCArKwudOnVCQEBAoQMmIiIi3buZ9q/0/xIzGsXNzQ1ZWVlo06YNAgMD8dlnn6FBgwZyxEZERERFlP1fnw095RnP963tG9zc3PDo0SPcuHEDN27cwM2bN/Ho0SM5YiMiIqIiepb9vLdEdVc7vcWgdbIRHx+P27dvY/LkycjOzsaUKVPg6uqKZs2aYeLEiXLESERERIWUlfO8ZsNcT51DgUL22XByckLnzp3RqlUrtGzZEtu3b8e6desQGxuL2bNn6zpGIiIiKqTzyRkAAHM9DXsFCpFsbN26FTExMYiJicHZs2dRpkwZ+Pv7Y9GiRQgKCpIjRiIiIiqkWw+edxC9nf5EbzFonWwMGTIErVu3xuDBgxEYGIj69evLERcRERHpgKpfaEi98nqLQetkIzU1VY44iIiISAbP/pvUq4ydpd5i0LoBJzAwED/++CP+/fff169MREREepX9XwdRM1P9jX3VOtnw9fXFhAkT4ObmhsGDB+Ovv/6SIy4iIiLSgfjEBwD0N1U5UIhkY8GCBbh16xZ+/PFH3LlzB61bt0bdunUxf/583L59W44YiYiIqBCeZOXgYurzubCszE31Fkeh0hxTU1N06dIF27Ztw61bt9CrVy9MmTIFlStXRteuXXHgwAFdx0lERERayniSJf0/tL6b3uIoUp3KsWPHMHXqVMyfPx/lypXDpEmTUK5cOXTq1AmffPKJrmIkIiKiQsj+r3OohamJXjuIFmo0yk8//YRVq1bh4sWL6NSpE9avX4+QkBAoFM87n/To0QNdu3bF/PnzdR4wERERFYwq2dBn51CgEMlGpUqVUL16dQwYMAD9+vWDq6urxjpNmzZFkyZNdBIgERERFU7Wfw9hM9PnU9hQiGRj//798Pf3f+U6Dg4OiI6OLnRQREREVHRHL98DAJjpcSQKUIg+G/7+/sjOzsbvv/+O7777Dg8fPgQAJCUl8emvREREBuT/fr8IAFAKodc4tE42rl+/jgYNGqBLly4ICwvDnTt3AABz587VW6fQt99+G87Ozujevbte9k9ERGSI/utKiUmhtfUah9bJxqhRo9C4cWOkpaXB2tpaKn/77bexf/9+nQZXUCNHjsSPP/6ol30TEREZqhzl8xqNRlWc9RqH1n02Dh8+jCNHjsDCwkKt3MPDA7du3dJZYNoICgpCTEyMXvZNRERkqLJyDKODqNY1G0qlEjk5ORrlN2/ehL29vdYBHDp0CJ06dYK7uzsUCgW2bdumsc7SpUtRrVo1WFlZwdfXF3/88YfW+yEiIjI2qpoN85LWQTQ4OBiLFy+WlhUKBR49eoTw8HB06NBB6wAyMzPh5eWFb775Js/XN2zYgNGjR2Py5MmIi4uDv78/QkNDcePGDa33RUREZCzuPXqKx8+eVw6YlrShr4sWLUJQUBDq1q2LJ0+eoFevXrh48SLKli2Ln3/+WesAQkNDERoamu/rCxcuxMCBAzFo0CAAwOLFi7F37158++23mDVrltb7e/r0KZ4+fSotZ2RkaL0NIiIiQ5aW+Qwt57x4dEiJm9TL3d0d8fHxWL9+PU6cOAGlUomBAwfigw8+UOswqgvPnj3DiRMnMHHiRLXy9u3b4+jRo4Xa5qxZszB9+nRdhEdERGSQbj34F0+ynvfX6OTlDlc9TlUOFKIZZc2aNbC2tkb//v3xzTffYOnSpRg0aBCsra0xfvx4nQZ39+5d5OTkoHz58mrl5cuXR0pKirQcEhKCd999F7t370alSpVw/PjxfLc5adIkpKenSz+JiYk6jZmIiEjfVB1DKzlbY8n7jaTHieiL1jUbw4cPh5OTEzp27KhWPmbMGKxfvx7z5s3TWXAqL58kIYRa2d69ewu8LUtLS1ha6jfDIyIikpOqY6i+R6GoaF2zsX79evTu3RuHDh2SykaMGIGNGzfqfIrysmXLwtTUVK0WA3j+MLiXazuIiIjouez/kg19dwxV0TrZePPNN7Fs2TJ07doVsbGxGDZsGLZs2YLo6GjUrq3bGcosLCzg6+uLqKgotfKoqCj4+fnpdF9ERESlhaEMeVXRuhkFAN577z2kpaWhVatWcHV1xcGDB1GjRo1CBfDo0SNcunRJWr569Sri4+Ph4uKCKlWqYOzYsejTpw8aN26MFi1aYPny5bhx4waGDh1aqP0RERGVZumPs3Dj/mMAhlOzUaBkY+zYsXmWlytXDo0aNcLSpUulsoULF2oVQGxsLIKCgjT21bdvX0RGRqJnz564d+8eZsyYgeTkZNSvXx+7d++Gh4eHVvshIiIq7VLSnyBgXjSeZhvGzKEqBUo24uLi8iyvXr06MjIypNcL09s1MDAQ4jVPoxs2bBiGDRum9bZfJSIiAhEREXnOhkpERFQSXbn7CE+zlTBRAC62Fni7UUV9hwQAUIjXfdKXchkZGXB0dER6ejocHBz0HQ4REVGh/XHxDvqsPIbabvbYM7q17Psr6GeoYfQcISIioiKThrzqecbQlzHZICIiKiVUyYapnifxehmTDSIiolLC0ObXUGGyQUREVEooS3Ky4ePjg7S0NADAjBkz8PjxY1mDIiIiooLJUQr8deUeDiTcxqmb6QAML9ko0NDX8+fPIzMzE87Ozpg+fTqGDh0KGxsbuWMjIiKi14g8eg1f7DqnVmYoM4eqFCjZ8Pb2Rv/+/dGqVSsIITB//nzY2dnlue7UqVN1GqBcOM8GERGVBjfTnrc2uNpbooKjFcxMFOjnV1W/Qb2kQPNsXLhwAeHh4bh8+TJOnjyJunXrwsxMM09RKBQ4efKkLIHKhfNsEBFRSRa+/QxW/3kdI9rUwLj2nsW674J+hhaoZsPT0xPr168HAJiYmGD//v0oV66cbiIlIiKiQlONQDExsOGuuWn9IDalUilHHERERFQIyv8aKAzlOSh5KdRTXy9fvozFixfj/PnzUCgUqFOnDkaNGoXq1avrOj4iIiJ6BdVEXiYGnGxo3V117969qFu3Lo4dO4aGDRuifv36+Pvvv1GvXj1ERUXJESMRERHlI+e/BgdDG+6am9Y1GxMnTsSYMWMwe/ZsjfJPP/0UwcHBOguOiIiIXk3VjGJoU5TnpnWycf78eWzcuFGjfMCAAVi8eLEuYiIiIqI8XLubiRPX09TL7mUCMOxmFK2TDVdXV8THx6NmzZpq5fHx8RyhQkREJKOey//E7Yyneb5maWZYE3nlpnWyMXjwYHz00Ue4cuUK/Pz8oFAocPjwYcyZMwfjxo2TI0ZZcFIvIiIqaVSJRos3ysAiV3LhYmuBN+u76Sus1yrQpF65CSGwePFiLFiwAElJSQAAd3d3jB8/HiNHjoTCgNuM8sJJvYiIqCRQKgXe+Gw3AODklGC42FroOSIdT+qVm0KhwJgxYzBmzBg8fPgQAGBvb1/4SImIiOi1cnLVDRhyZ9C8FGqeDRUmGURERMVDNZ8GAJgYbveMPJWwcImIiIxT7k4Phjw1eV6YbBAREZUAas0oBjzMNS9MNoiIiEoAtWaU0lyzkZWVhaCgIPzzzz9yxUNERER5UKolG3oMpBC06iBqbm6OM2fOlLjhrURERIbu8bNsZGXnPxvFg3+zpP+XtGYUrUejfPjhh1i5cqXGs1GIiIiocHaeSsKYDfHIVr5+6iuFAiXuS7/WycazZ8+wYsUKREVFoXHjxrC1tVV7feHChToLTk6cQZSIiAzF8Wv3C5RoAEDb2uVljkb3tE42zpw5Ax8fHwDQ6LtRkjKtsLAwhIWFSbOfERER6Yvqya0j2tTAyLY1X7muuWnJG9uhdbIRHR0tRxxERERGK0f5/F9zU5MSmUy8TqGP6NKlS9i7dy/+/fdfAM+fmUJERETaU32GlrB+nwWmdbJx7949tG3bFrVq1UKHDh2QnJwMABg0aFCJeuorERGRoVA1o5Sk7gja0DrZGDNmDMzNzXHjxg3Y2NhI5T179sSePXt0GhwREZExUDWjlLQhrQWldZ+Nffv2Ye/evahUqZJaec2aNXH9+nWdBUZERGQs2IzykszMTLUaDZW7d+/C0tJSJ0EREREZE6WUbJTObEPrZKN169b48ccfpWWFQgGlUol58+YhKChIp8EREREZg5z/xliU1mRD62aUefPmITAwELGxsXj27BkmTJiAs2fP4v79+zhy5IgcMRIREZVqSjajqKtbty5Onz6Npk2bIjg4GJmZmejWrRvi4uJQvXp1OWIkIiIq1aQ+G6U029C6ZgMA3NzcMH36dF3HQkREVGLtOJWEBfsuIDtH+3mn7mU+BcBmFDVpaWlYuXIlzp8/D4VCgTp16qB///5wcXHRdXyy4bNRiIhIl345cRPX7z0u0jbecLV9/UolkEJoOfXnwYMH0aVLFzg4OKBx48YAgBMnTuDBgwfYsWMHAgICZAlULqpno6Snp8PBwUHf4RARUQnVe8XfOHzpLsYF10KAp6vW73e2sUBlF83RnoasoJ+hWtdshIWFoUePHvj2229hamoKAMjJycGwYcMQFhaGM2fOFD5qIiKiEkrVydOjrC0aVnLSbzAGRusOopcvX8a4ceOkRAMATE1NMXbsWFy+fFmnwREREZUUpX1ESVFonWz4+Pjg/PnzGuXnz5+Ht7e3LmIiIiIqcZSlfK6MoihQM8rp06el/48cORKjRo3CpUuX0Lx5cwDAX3/9hYiICMyePVueKImIiAycqgskUw1NBUo2vL29oVAo1B4jP2HCBI31evXqhZ49e+ouOiIiohJC9RFZWp/cWhQFSjauXr0qdxxEREQlGvts5K9AyYaHh4fccRAREZVo7LORv0JN6nXr1i0cOXIEqampUCqVaq+NHDlSJ4ERERGVJC+mHNdzIAZI62Rj1apVGDp0KCwsLFCmTBm1timFQsFkg4iIjJKSfTbypXWyMXXqVEydOhWTJk2CCdM3IiIiALn7bDDZeJnW2cLjx4/x3nvvMdEgIiLK5UWfDf3GYYi0zhgGDhyITZs2yRELERFRifVing1mGy/Tuhll1qxZ6NixI/bs2YMGDRrA3Nxc7fWFCxfqLDgiIqLCOH7tPob+dAIPn2YX2z6fZT8fMMGaDU1aJxszZ87E3r174enpCQAaHURLCj5inoio9Dr0zx3cy3xW7Pu1szRDjXJ2xb5fQ6f1I+adnZ2xaNEi9OvXT6aQihcfMU9EVPrM25uAiOjL6O5bCWOCaxXbfp1tzGFjUahZJUok2R4xb2lpiZYtWxYpOCIiIjmpvkbbW5mhopO1foMh7TuIjho1CkuWLJEjFiIiIp3gbJ6GReuajWPHjuHAgQPYtWsX6tWrp9FBdMuWLToLjoiIqDAE+ARWQ6J1suHk5IRu3brJEQsREZFOqJpRTDg0xCAUarpyIiIiQ/ZizgsyBJwGlIiISh0+p8SwaF2zUa1atVdevCtXrhQpICIioqISUrKh3zjoOa2TjdGjR6stZ2VlIS4uDnv27MH48eN1FRcREVGhvXgomp4DIQCFSDZGjRqVZ3lERARiY2OLHBAREZGu8DklhkFnfTZCQ0OxefNmXW2OiIio0FizYVh0lmz88ssvcHFx0dXmiIiICk16EAc7bRgErZtRGjVqpNZBVAiBlJQU3LlzB0uXLtVpcERERIXBmg3DonWy0bVrV7VlExMTuLq6IjAwELVr19ZVXERERIUmVWywz4ZB0DrZCA8PlyMOIiIinRGs2TAonNSLiIhKHc6zYVgKXLNhYmLy2pnYFAoFsrOzixwUERFRUaj6bHAGUcNQ4GRj69at+b529OhRLFmyRKq2IiKikik14wmm7TyLtMwsfYdSJJfuPALAmg1DUeBko0uXLhplCQkJmDRpEnbu3IkPPvgAX3zxhU6Dk1NERAQiIiKQk5Oj71CIiAzG3nO3sft/KfoOQ2fcHKz0HQKhEB1EASApKQnh4eFYvXo1QkJCEB8fj/r16+s6NlmFhYUhLCwMGRkZcHR01Hc4REQGIStbCQDwqeKEfi2r6TmaonG0NkerGmX1HQZBy2QjPT0dM2fOxJIlS+Dt7Y39+/fD399frtiIiKiYqRrDKznboLOXu15jodKjwMnG3LlzMWfOHLi5ueHnn3/Os1mFiIhKNiF1rNRzIFSqFDjZmDhxIqytrVGjRg2sXr0aq1evznO9LVu26Cw4IiLSD+YapEsFTjY+/PBDDiEiIirlXkzzzb/3pDsFTjYiIyNlDIOIiAzBiweY6TUMKmU4gygREUn4TBGSA5MNIiKScJpvkgOTDSIikoj/6jaYa5AuMdkgIiIJazZIDkw2iIhIA/tskC4x2SAiIgkn9SI5MNkgIiIJm1FIDkw2iIhIIqT/Mdsg3WGyQUREEtZskByYbBARkYRDX0kOTDaIiEjCmg2SA5MNIiKSSKNRWLdBOsRkg4iIJKoOoibMNUiHmGwQEZHkRTMKsw3SHSYbREQkEbkGvxLpCpMNIiKSsIMoyYHJBhERSVT1GuwgSrrEZIOIiCSs2SA5MNkgIiIJJ/UiOZjpOwAiMh6rj17DyRtp+g6DXuFsUgYA1myQbhltshEREYGIiAjk5OToOxQio5CW+QzhO87qOwwqIGdbC32HQKWIQqimizNSGRkZcHR0RHp6OhwcHPQdDlGpdTvjCZrN3A+FApjcoY6+w6FXsLM0Q0cvd9hZGu33USqggn6G8jeJiIqF6muNqUKBQf5v6DcYIipW7CBKRMWCk0URGS8mG0RUrNjxkMj4MNkgomIhzd/AQZVERofJBhEVCzaiEBkvJhtEVLxYsUFkdJhsEFGxUI2yZ65BZHyYbBAREZGsmGwQUbHgA76IjBeTDSIiIpIVkw0iKlYc+kpkfJhsEFGxYDMKkfFiskFExYLTlRMZLyYbRFSsWLFBZHyYbBBRsXjRjMJ0g8jYMNkgIiIiWTHZIKJioeqxwXoNIuPDZIOIioVqunIiMj5MNoioeLFqg8joMNkgomLBZhQi48Vkg4iIiGTFZIOIigWHvhIZLyYbRFRM2EGUyFgx2SCiYsWKDSLjw2SDiIqF1Iyi3zCISA+YbBBRsWAjCpHxYrJBRMWKHUSJjA+TDSIqFmxGITJeTDaIiIhIVkw2iKhYiP96bbAVhcj4MNkgomLB57ARGS8mG0RUzFi1QWRsmGwQUbF4MV25fuMgouLHZIOIioXgTBtERovJBhEVK1ZsEBkfJhtEVCzYQZTIeDHZIKJixT4bRMaHyQYRFSsFG1KIjA6TDSIqFmxGITJepSLZ2LVrFzw9PVGzZk2sWLFC3+EQ0SuwGYXI+JjpO4Ciys7OxtixYxEdHQ0HBwf4+PigW7ducHFx0XdoRJSLNF25nuMgouJX4ms2jh07hnr16qFixYqwt7dHhw4dsHfvXn2HRURERP/Re7Jx6NAhdOrUCe7u7lAoFNi2bZvGOkuXLkW1atVgZWUFX19f/PHHH9JrSUlJqFixorRcqVIl3Lp1qzhCJyItvJhBlHUbRMZG780omZmZ8PLyQv/+/fHOO+9ovL5hwwaMHj0aS5cuRcuWLfHdd98hNDQU586dQ5UqVSDy6HWmzz9mtzOeIO5Gmt72T2Sort59rO8QiEhP9J5shIaGIjQ0NN/XFy5ciIEDB2LQoEEAgMWLF2Pv3r349ttvMWvWLFSsWFGtJuPmzZto1qxZvtt7+vQpnj59Ki1nZGTo4CheiLvxAEPXnNTpNolKEzNT1mwQGRu9Jxuv8uzZM5w4cQITJ05UK2/fvj2OHj0KAGjatCnOnDmDW7duwcHBAbt378bUqVPz3easWbMwffp02WJ2sjFHYw9n2bZPVJIpFECPxpX1HQYRFTODTjbu3r2LnJwclC9fXq28fPnySElJAQCYmZlhwYIFCAoKglKpxIQJE1CmTJl8tzlp0iSMHTtWWs7IyEDlyrr749f8jTL45WM/nW2PiIiopDPoZEPl5T4YQgi1ss6dO6Nz584F2palpSUsLS11Gh8RERHlT++jUV6lbNmyMDU1lWoxVFJTUzVqO4iIiMgwGXSyYWFhAV9fX0RFRamVR0VFwc+PTRVEREQlgd6bUR49eoRLly5Jy1evXkV8fDxcXFxQpUoVjB07Fn369EHjxo3RokULLF++HDdu3MDQoUP1GDUREREVlN6TjdjYWAQFBUnLqs6bffv2RWRkJHr27Il79+5hxowZSE5ORv369bF79254eHjoK2QiIiLSgkLkNSuWEYiIiEBERARycnLwzz//ID09HQ4ODvoOi4iIqMTIyMiAo6Pjaz9DjTbZUCnoiSIiIiJ1Bf0MNegOokRERFTyMdkgIiIiWTHZICIiIlkx2SAiIiJZ6X3oq76p+sfq+umvREREpZ3qs/N1Y02MPtl4+PAhAOj0YWxERETG5OHDh3B0dMz3daMf+qpUKpGUlAR7e3uNB74VlupJsomJiRxOa0B4XQwXr43h4rUxTIZyXYQQePjwIdzd3WFikn/PDKOv2TAxMUGlSpVk2baDgwNvTgPE62K4eG0MF6+NYTKE6/KqGg0VdhAlIiIiWTHZICIiIlkx2ZCBpaUlwsPDYWlpqe9QKBdeF8PFa2O4eG0MU0m7LkbfQZSIiIjkxZoNIiIikhWTDSIiIpIVkw0iIiKSFZMNIiIikhWTjUJaunQpqlWrBisrK/j6+uKPP/545foHDx6Er68vrKys8MYbb2DZsmXFFKlx0ea6xMTEQKFQaPwkJCQUY8TG4dChQ+jUqRPc3d2hUCiwbdu2176H94z8tL0uvGeKx6xZs9CkSRPY29ujXLly6Nq1Ky5cuPDa9xnyPcNkoxA2bNiA0aNHY/LkyYiLi4O/vz9CQ0Nx48aNPNe/evUqOnToAH9/f8TFxeGzzz7DyJEjsXnz5mKOvHTT9rqoXLhwAcnJydJPzZo1iyli45GZmQkvLy988803BVqf90zx0Pa6qPCekdfBgwcRFhaGv/76C1FRUcjOzkb79u2RmZmZ73sM/p4RpLWmTZuKoUOHqpXVrl1bTJw4Mc/1J0yYIGrXrq1WNmTIENG8eXPZYjRG2l6X6OhoAUCkpaUVQ3SkAkBs3br1levwnil+BbkuvGf0IzU1VQAQBw8ezHcdQ79nWLOhpWfPnuHEiRNo3769Wnn79u1x9OjRPN/z559/aqwfEhKC2NhYZGVlyRarMSnMdVFp1KgRKlSogLZt2yI6OlrOMKmAeM8YNt4zxSs9PR0A4OLiku86hn7PMNnQ0t27d5GTk4Py5curlZcvXx4pKSl5viclJSXP9bOzs3H37l3ZYjUmhbkuFSpUwPLly7F582Zs2bIFnp6eaNu2LQ4dOlQcIdMr8J4xTLxnip8QAmPHjkWrVq1Qv379fNcz9HvG6J/6WlgvP45eCPHKR9TntX5e5VQ02lwXT09PeHp6SsstWrRAYmIi5s+fj9atW8saJ70e7xnDw3um+A0fPhynT5/G4cOHX7uuId8zrNnQUtmyZWFqaqrxbTk1NVUjq1Rxc3PLc30zMzOUKVNGtliNSWGuS16aN2+Oixcv6jo80hLvmZKD94x8RowYgR07diA6OhqVKlV65bqGfs8w2dCShYUFfH19ERUVpVYeFRUFPz+/PN/TokULjfX37duHxo0bw9zcXLZYjUlhrkte4uLiUKFCBV2HR1riPVNy8J7RPSEEhg8fji1btuDAgQOoVq3aa99j8PeMHjunlljr168X5ubmYuXKleLcuXNi9OjRwtbWVly7dk0IIcTEiRNFnz59pPWvXLkibGxsxJgxY8S5c+fEypUrhbm5ufjll1/0dQilkrbXZdGiRWLr1q3in3/+EWfOnBETJ04UAMTmzZv1dQil1sOHD0VcXJyIi4sTAMTChQtFXFycuH79uhCC94y+aHtdeM8Uj48//lg4OjqKmJgYkZycLP08fvxYWqek3TNMNgopIiJCeHh4CAsLC+Hj46M2JKlv374iICBAbf2YmBjRqFEjYWFhIapWrSq+/fbbYo7YOGhzXebMmSOqV68urKyshLOzs2jVqpX49ddf9RB16acaMvnyT9++fYUQvGf0RdvrwnumeOR1TQCIVatWSeuUtHuGj5gnIiIiWbHPBhEREcmKyQYRERHJiskGERERyYrJBhEREcmKyQYRERHJiskGERERyYrJBhEREcmKyQYREVEpdejQIXTq1Anu7u5QKBTYtm2b1tsQQmD+/PmoVasWLC0tUblyZcycOVOrbTDZIKICuXbtGhQKBeLj4/UdiiQhIQHNmzeHlZUVvL299R0OkcHJzMyEl5cXvvnmm0JvY9SoUVixYgXmz5+PhIQE7Ny5E02bNtVqG0w2iEqIfv36QaFQYPbs2Wrl27ZtM4hHSOtDeHg4bG1tceHCBezfv1/f4RAZnNDQUHz55Zfo1q1bnq8/e/YMEyZMQMWKFWFra4tmzZohJiZGev38+fP49ttvsX37dnTu3BnVqlWDt7c32rVrp1UcTDaIShArKyvMmTMHaWlp+g5FZ549e1bo916+fBmtWrWCh4eHQTxGW1tFOXYiXejfvz+OHDmC9evX4/Tp03j33Xfx5ptv4uLFiwCAnTt34o033sCuXbtQrVo1VK1aFYMGDcL9+/e12g+TDaISpF27dnBzc8OsWbPyXWfatGkaTQqLFy9G1apVpeV+/fqha9eumDlzJsqXLw8nJydMnz4d2dnZGD9+PFxcXFCpUiX88MMPGttPSEiAn58frKysUK9ePbVvQQBw7tw5dOjQAXZ2dihfvjz69OmDu3fvSq8HBgZi+PDhGDt2LMqWLYvg4OA8j0OpVGLGjBmoVKkSLC0t4e3tjT179kivKxQKnDhxAjNmzIBCocC0adPy3I5qf8OHD4eTkxPKlCmDzz//HLkfC7VmzRo0btwY9vb2cHNzQ69evZCamiq9HhMTA4VCgV9//RVeXl6wsrJCs2bN8L///U9tX0ePHkXr1q1hbW2NypUrY+TIkcjMzJRer1q1Kr788kv069cPjo6OGDx4MJ49e4bhw4ejQoUKsLKyQtWqVV95fYl05fLly/j555+xadMm+Pv7o3r16vjkk0/QqlUrrFq1CgBw5coVXL9+HZs2bcKPP/6IyMhInDhxAt27d9dqX0w2iEoQU1NTzJw5E0uWLMHNmzeLtK0DBw4gKSkJhw4dwsKFCzFt2jR07NgRzs7O+PvvvzF06FAMHToUiYmJau8bP348xo0bh7i4OPj5+aFz5864d+8eACA5ORkBAQHw9vZGbGws9uzZg9u3b6NHjx5q21i9ejXMzMxw5MgRfPfdd3nG93//939YsGAB5s+fj9OnTyMkJASdO3eWvnElJyejXr16GDduHJKTk/HJJ5/ke6yq/f3999/4+uuvsWjRIqxYsUJ6/dmzZ/jiiy9w6tQpbNu2DVevXkW/fv00tjN+/HjMnz8fx48fR7ly5dC5c2dkZWUBAP73v/8hJCQE3bp1w+nTp7FhwwYcPnwYw4cPV9vGvHnzUL9+fZw4cQJTpkzB119/jR07dmDjxo24cOEC1qxZo5YYEsnl5MmTEEKgVq1asLOzk34OHjyIy5cvA3ie9D99+hQ//vgj/P39ERgYiJUrVyI6OhoXLlwo+M70+sxZIiqwvn37ii5dugghhGjevLkYMGCAEEKIrVu3ity3cnh4uPDy8lJ776JFi4SHh4fatjw8PEROTo5U5unpKfz9/aXl7OxsYWtrK37++WchhBBXr14VAMTs2bOldbKyskSlSpXEnDlzhBBCTJkyRbRv315t34mJiQKAuHDhghBCiICAAOHt7f3a43V3dxdfffWVWlmTJk3EsGHDpGUvLy8RHh7+yu0EBASIOnXqCKVSKZV9+umnok6dOvm+59ixYwKAePjwoRDixaPY169fL61z7949YW1tLTZs2CCEEKJPnz7io48+UtvOH3/8IUxMTMS///4rhBDCw8NDdO3aVW2dESNGiDZt2qjFRyQHAGLr1q3S8vr164WpqalISEgQFy9eVPtJTk4WQggxdepUYWZmpradx48fCwBi3759Bd43azaISqA5c+Zg9erVOHfuXKG3Ua9ePZiYvPgTUL58eTRo0EBaNjU1RZkyZdSaEwCgRYsW0v/NzMzQuHFjnD9/HgBw4sQJREdHq31Lql27NgBI35QAoHHjxq+MLSMjA0lJSWjZsqVaecuWLaV9aaN58+ZqnWhbtGiBixcvIicnBwAQFxeHLl26wMPDA/b29ggMDAQA3LhxQ207uY/dxcUFnp6easceGRmpduwhISFQKpW4evVqvsfer18/xMfHw9PTEyNHjsS+ffu0Pj6iwmjUqBFycnKQmpqKGjVqqP24ubkBeH7PZWdnq92///zzDwDAw8OjwPsy023oRFQcWrdujZCQEHz22Wca1f0mJiZq/REASFX9uZmbm6stKxSKPMuUSuVr41F9kCuVSnTq1Alz5szRWKdChQrS/21tbV+7zdzbVRFC6HzkTWZmJtq3b4/27dtjzZo1cHV1xY0bNxASElKgDpy5j33IkCEYOXKkxjpVqlSR/v/ysfv4+ODq1av47bff8Pvvv6NHjx5o164dfvnllyIeGRHw6NEjXLp0SVq+evUq4uPj4eLiglq1auGDDz7Ahx9+iAULFqBRo0a4e/cuDhw4gAYNGqBDhw5o164dfHx8MGDAACxevBhKpRJhYWEIDg5GrVq1ChwHkw2iEmr27Nnw9vbWuOFdXV2RkpKi9sGsy7kx/vrrL7Ru3RoAkJ2djRMnTkj9Enx8fLB582ZUrVoVZmaF//Pi4OAAd3d3HD58WNoX8LwDprbj+1Uxv7xcs2ZNmJqaIiEhAXfv3sXs2bNRuXJlAEBsbGy+21ElDmlpafjnn3+kmhsfHx+cPXsWNWrU0Do+BwcH9OzZEz179kT37t3x5ptv4v79+3BxcdF6W0S5xcbGIigoSFoeO3YsAKBv376IjIzEqlWr8OWXX2LcuHG4desWypQpgxYtWqBDhw4Ann952blzJ0aMGIHWrVvD1tYWoaGhWLBggVZxMNkgKqEaNGiADz74AEuWLFErDwwMxJ07dzB37lx0794de/bswW+//QYHBwed7DciIgI1a9ZEnTp1sGjRIqSlpWHAgAEAgLCwMHz//fd4//33MX78eJQtWxaXLl3C+vXr8f3338PU1LTA+xk/fjzCw8NRvXp1eHt7Y9WqVYiPj8fatWu1jjkxMRFjx47FkCFDcPLkSSxZskT6Y1mlShVYWFhgyZIlGDp0KM6cOYMvvvgiz+3MmDEDZcqUQfny5TF58mSULVsWXbt2BQB8+umnaN68OcLCwjB48GDY2tri/PnziIqK0rhGuS1atAgVKlSAt7c3TExMsGnTJri5ucHJyUnr4yR6WWBgoEZNZ27m5uaYPn06pk+fnu867u7u2Lx5c5HiYJ8NohLsiy++0PhDUqdOHSxduhQRERHw8vLCsWPHXjlSQ1uzZ8/GnDlz4OXlhT/++APbt29H2bJlATz/o3TkyBHk5OQgJCQE9evXx6hRo+Do6KjWP6QgRo4ciXHjxmHcuHFo0KAB9uzZgx07dqBmzZpax/zhhx/i33//RdOmTREWFoYRI0bgo48+AvC8JigyMhKbNm1C3bp1MXv2bMyfPz/fYx81ahR8fX2RnJyMHTt2wMLCAgDQsGFDHDx4EBcvXoS/vz8aNWqEKVOmqDUf5cXOzg5z5sxB48aN0aRJE1y7dg27d+/W+nwRGTKFeFXKQ0RUwgUGBsLb2xuLFy8u9DZiYmIQFBSEtLQ01jgQFQJTZyIiIpIVkw0iIiKSFZtRiIiISFas2SAiIiJZMdkgIiIiWTHZICIiIlkx2SAiIiJZMdkgIiIiWTHZICIiIlkx2SAiIiJZMdkgIiIiWf0/ZGyAGbB5MCkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "AI_OA = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/OA_Kwds_AI.parquet\")\n",
    "counts = sorted([el[0] for el in AI_OA.select(\"Kwd_count\").collect()])\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.figure(figsize=(6,4))\n",
    "plt.semilogy(np.linspace(0,AI_OA.count(), len(counts)), counts)\n",
    "plt.xlabel(\"Number of papers\")\n",
    "plt.ylabel(\"Number of keywords\")\n",
    "plt.title(\"Cumulative number of papers in OpenAIRE\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1037658c-a927-4243-bf78-66e219cba8c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications in S2 with 1+ Kwds 2128190\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications in S2 with 3+ Kwds 608155\n",
      "Number of publications in OA with 1+ Kwds 2058409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 13:>                                                       (0 + 40) / 41]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications in OA with 3+ Kwds 574346\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "AI_S2 = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/S2_Kwds_AI.parquet\")\n",
    "print(\"Number of publications in S2 with 1+ Kwds\", AI_S2.count())\n",
    "AI_S2 = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/S2_Kwds3_AI.parquet\")\n",
    "print(\"Number of publications in S2 with 3+ Kwds\", AI_S2.count())\n",
    "AI_OA = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/OA_Kwds_AI.parquet\")\n",
    "print(\"Number of publications in OA with 1+ Kwds\", AI_OA.count())\n",
    "AI_OA = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/OA_Kwds3_AI.parquet\")\n",
    "print(\"Number of publications in OA with 3+ Kwds\", AI_OA.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "00dd83b6-480b-4810-a6c6-9542f9bcf030",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAGHCAYAAAAZa5AYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABHmUlEQVR4nO3deVhU1f8H8PewDTuCIMgikpqCG6m44ALkQhiamWlp7pom5pqllaLWV9y1/GFZmmSZWmluKYp7iuaKG2pqKii4Iwgqy8z5/aFzYwRhBmecYXi/noen5twz937uAZwPZ7syIYQAERERkZ6YGToAIiIiMm1MNoiIiEivmGwQERGRXjHZICIiIr1iskFERER6xWSDiIiI9IrJBhEREekVkw0iIiLSKyYbREREpFdMNiq4EydOoH///vDz84O1tTXs7e3RqFEjzJw5E3fv3jV0eCWaPHkyZDJZmd67adMmTJ48udhj1atXR79+/coemAnp168f7O3tDR2GxlatWoW6devCxsYGMpkMSUlJL+zaiYmJmDx5Mu7du6f3az148ACTJ0/Grl27dH7uqVOnIiAgAEqlUiqTyWRqX05OTggNDcWff/5Z5P23b9+GXC6HTCbD4cOHpXLV72tpX6GhoQAAIQRWrlyJ1q1bo0qVKrC2toa3tzfCw8OxePFiAI9/PjU5Z+HfZ13F17t3b3Tp0kV3DW/qBFVY3333nbCwsBB169YVsbGxYufOnWLr1q1i2rRpws/PT3Tp0sXQIZYoOjpalPVHOCoq6pnvPXr0qLhw4cLzhGYy+vbtK+zs7AwdhkZu3rwpLC0tRadOncSuXbvE/v37RU5Ozgu7/qxZswQAcenSJb1f69atWwKAiI6O1ul5r127Juzs7MRvv/2mVg5AdOvWTezfv1/s27dP/PTTT6J27dpCJpOJjRs3qtWdO3euACAAiKFDh0rlqampYv/+/dLXmjVrBADx4YcfqpWfPn1aCCHEJ598IgCIwYMHi3Xr1okdO3aIpUuXit69e4vIyEghhBAXLlxQe29sbKwAIKZNm6ZWXvj3WVfxXbhwQVhYWIjt27fr9HtgqphsVFCJiYnC3NxcvPbaa+LRo0dFjufm5op169YZIDLN6SvZoP+8qGRDF0nB3r17BQCxatUqHUSkPVNINj7++GPh5eUlFAqFWjkAERUVpVZ24cIFAUC0a9dOrbxevXqiSpUqIigoSDg5OYkHDx4Ue61Lly4JAGLWrFlFjj148EDI5XLRp0+fYt/7dHwqO3fuFACKJEu6jk8lMjJStG/f/pnH6T8cRqmgpk2bBplMhu+++w5yubzIcSsrK3Tu3Fl6LZPJih12eHrIIS4uDjKZDDt27MDgwYNRuXJlODo6ok+fPsjJycH169fRvXt3VKpUCVWrVsVHH32E/Px86f27du2CTCYr0j18+fJlyGQyxMXFlXhfq1atQocOHVC1alXY2NjA398f48ePR05OjlSnX79+iI2Nle5L9XX58uUi93Tr1i1YWVlh4sSJRa519uxZyGQyfP3111LZ9evXMWTIEHh7e8PKygp+fn6YMmUKCgoKSoxbdd3IyEjEx8ejUaNGsLGxQZ06dfDDDz+o1XvW8JGq7VX3UficGzduxCuvvCK1ycaNG6X3+Pv7w87ODk2bNlXrVi7s9OnTaNu2Lezs7ODm5obhw4fjwYMHanWEEFi4cCECAwNhY2MDZ2dndOvWDf/++69avdDQUNSrVw979uxBcHAwbG1tMWDAgBLbZv369WjRogVsbW3h4OCA9u3bY//+/dLxfv36oVWrVgCAHj16qHV3F0fVVgkJCejfvz9cXFxgZ2eHTp06FYk3ISEBb7zxBry9vWFtbY2aNWtiyJAhuH37tlRn8uTJGDduHADAz89P+pkq/HO8atUqtGjRAnZ2drC3t0d4eDiOHTumdi3VsNWFCxfQsWNH2Nvbw8fHB2PHjkVubi6Ax78Lbm5uAIApU6YUGSq4desW3n//ffj4+EAul8PNzQ0tW7bEtm3bSmzjvLw8LFmyBD179oSZWekfDTVq1ICbmxuuXLkilf399984deoUevfujcGDByMzMxOrV68u9VxPy8nJQW5uLqpWrVrscU3iK46u4lPp3bs3tm3bhosXL5b5HBUFk40KSKFQYMeOHWjcuDF8fHz0co1BgwbByckJK1euxOeff45ffvkFgwcPxuuvv46GDRvi999/R9++fTFnzhwsWLBAZ9c9f/48OnbsiCVLliA+Ph6jRo3Cr7/+ik6dOkl1Jk6ciG7dugEA9u/fL30V9w+bm5sbIiMj8eOPP6qNYQPA0qVLYWVlhV69egF4nGg0bdoUW7ZswaRJk7B582YMHDgQMTExGDx4sEbxHz9+HGPHjsXo0aOxbt06NGjQAAMHDsSePXvK2iQ4fvw4JkyYgE8++QRr1qyBk5MTunbtiujoaCxevBjTpk3D8uXLkZmZicjISDx8+FDt/fn5+ejYsSPatm2LtWvXYvjw4Vi0aBF69OihVm/IkCEYNWoU2rVrh7Vr12LhwoU4ffo0goODcePGDbW66enpeO+999CzZ09s2rQJw4YNe2b8v/zyC9544w04OjpixYoVWLJkCTIyMhAaGoq9e/cCePw9VSWQ06ZNw/79+7Fw4cJS22bgwIEwMzPDL7/8gvnz5+PgwYMIDQ1Vm3dx8eJFtGjRAt988w22bt2KSZMm4e+//0arVq2kRHnQoEH48MMPAQBr1qyRfqYaNWokxfTuu+8iICAAv/76K3766Sfcv38frVu3RnJycpH27ty5M9q2bYt169ZhwIABmDdvHmbMmAEAqFq1KuLj46X4VddSJcS9e/fG2rVrMWnSJGzduhWLFy9Gu3btcOfOnRLb4u+//8adO3cQFhZWarsBQEZGBu7cuSMlPgCwZMkSAMCAAQPwzjvvwNbWVirThqurK2rWrImFCxdi7ty5OHv2LIQOHlCuq/hUQkNDIYTApk2bnjs2k2fgnhUygOvXrwsA4p133tH4PXhGl62vr6/o27ev9Hrp0qXSOGdhXbp0EQDE3Llz1coDAwNFo0aNpNeqbtCdO3eq1VN1aS5dulQqK20YRalUivz8fLF7924BQBw/flw6VtIwytP3tH79egFAbN26VSorKCgQnp6e4q233pLKhgwZIuzt7cWVK1fUzjd79mwBQBrrfRZfX19hbW2t9v6HDx8KFxcXMWTIkFLvW9X2hbvxfX19hY2Njbh69apUlpSUJACIqlWrqg1frF27VgAQ69evl8r69u0rAIivvvpK7Vr/+9//BACxd+9eIYQQ+/fvFwDEnDlz1OqlpqYKGxsb8fHHH0tlISEhAoBGY90KhUJ4enqK+vXrq3Wd379/X1SpUkUEBwdLZZp0oauo2urNN99UK9+3b58AIL788sti36f6mbpy5YoAoDbU+KxhlJSUFGFhYVHkd+L+/fvCw8NDdO/eXSpTtfevv/6qVrdjx46idu3a0uuShlHs7e3FqFGjSrz/4syYMUMAENevXy9yDIAYNmyYyM/PF3l5eeLMmTMiIiJCABCxsbFCiMdDYY6OjqJ58+Zq9yOTyYqdA1XaMMXBgwdFtWrVpPkVDg4OIjIyUixbtkwolcpi31PSz4Cu41Px8vISPXr0KLEOcRiF9CQyMlLttb+/PwDg9ddfL1JeuBv2ef3777/o2bMnPDw8YG5uDktLS4SEhAAAzpw5U6ZzRkREwMPDA0uXLpXKtmzZgrS0NLXu/40bNyIsLAyenp4oKCiQviIiIgAAu3fvLvVagYGBqFatmvTa2toaL7/88nO1UWBgILy8vKTXqu9FaGgobG1ti5QXdy1V741Kz549AQA7d+4E8PjeZTIZ3nvvPbV79/DwQMOGDYsMizk7O+PVV18tNfZz584hLS0NvXv3Vus6t7e3x1tvvYUDBw4UGc7RxtP3FRwcDF9fX+m+AODmzZsYOnQofHx8YGFhAUtLS/j6+gLQ7Gdqy5YtKCgoQJ8+fdTaxtraGiEhIUXaRiaTqfXEAUCDBg00/hlo2rQp4uLi8OWXX+LAgQNqw5QlSUtLg0wmg6ura7HHFy5cCEtLS1hZWcHf3x+JiYmYOnWq1Cv166+/IisrS+13YsCAARBCqP3uaCooKAgXLlxAfHw8Pv30U7Ro0QLbt29Hnz590LlzZ617OnQdn0qVKlVw7dq1Mr+/orAwdAD04rm6usLW1haXLl3S2zVcXFzUXltZWT2z/NGjRzq5ZnZ2Nlq3bg1ra2t8+eWXePnll2Fra4vU1FR07dq1yPCApiwsLNC7d28sWLAA9+7dQ6VKlRAXF4eqVasiPDxcqnfjxg1s2LABlpaWxZ6n8Bj/s1SuXLlImVwuL3PsgHbfCwBFvh8WFhZF4vLw8AAAqWv+xo0bEELA3d292BheeukltdfPGot/mur8xdX39PSEUqlERkaGWtKkDdV9PF2muq5SqUSHDh2QlpaGiRMnon79+rCzs4NSqUTz5s01+r6ohpCCgoKKPf70/ANbW1tYW1urlcnlco1/T1atWoUvv/wSixcvxsSJE2Fvb48333wTM2fOLPZ+VR4+fAhLS0uYm5sXe7x79+4YN24cZDIZHBwcUKNGDbW6S5YsgbW1NV577TVpGKpBgwaoXr064uLiMGXKlGee+1ksLS0RHh4u/Z7duXMH3bp1w8aNG7F582Z07NhR43PpIz7g8R8Ez/P7WVEw2aiAzM3N0bZtW2zevBlXr16Ft7d3qe+Ry+XSBLXCShsH1pbqH9mnr6XJB/WOHTuQlpaGXbt2Sb0ZAHSy70H//v0xa9YsrFy5Ej169MD69esxatQotX+cXF1d0aBBA/zvf/8r9hyenp7PHQeg3kaFJ/dq0kZlUVBQgDt37qglHNevXwfwX3Lk6uoKmUyGv/76q9gJx0+Xabo/iur86enpRY6lpaXBzMwMzs7Omt1IMVT38XRZzZo1AQCnTp3C8ePHERcXh759+0p1Lly4oPE1VD0Fv//+u9Qjok+urq6YP38+5s+fj5SUFKxfvx7jx4/HzZs3pbkez3pfXl4ecnJyYGdnV+S4m5sbmjRpUux7//nnH2n+TOGeucK2bNmiVXJQnMqVK2PUqFHYtWsXTp06pfH59Bnf3bt3Ub16da3fV9Ew2aigJkyYgE2bNmHw4MFYt26d9FetSn5+PuLj46Xu3OrVq+PEiRNqdXbs2IHs7GydxqX6pT1x4oRar8H69etLfa/qA+zpD7ZFixYVqauq8/DhQ9jY2JR6bn9/fzRr1gxLly6FQqFAbm4u+vfvr1YnMjISmzZtQo0aNZ7rA7A0hduo8F/LGzZs0Ns1ly9fjhEjRkivf/nlFwCQVnxERkZi+vTpuHbtGrp3766z69auXRteXl745Zdf8NFHH0nf45ycHKxevVpaoVJWy5cvx1tvvSW9TkxMxJUrVzBo0CAAZf+ZKiw8PBwWFha4ePGi2rWex7Ou9bRq1aph+PDh2L59O/bt21di3Tp16gB4PCG2QYMGWsWjmmT5/fffS4maysOHD/HGG2/ghx9+0PjDPD8/H1lZWcX29KmGrrRJ3nUdn0pBQQFSU1OfO4mqCJhsVFCq2fXDhg1D48aN8cEHH6Bu3brIz8/HsWPH8N1336FevXpSstG7d29MnDgRkyZNQkhICJKTk/F///d/cHJy0mlcHh4eaNeuHWJiYuDs7AxfX19s374da9asKfW9wcHBcHZ2xtChQxEdHQ1LS0ssX74cx48fL1K3fv36AIAZM2YgIiIC5ubmaNCgQZGkq7ABAwZgyJAhSEtLQ3BwMGrXrq12fOrUqUhISEBwcDBGjBiB2rVr49GjR7h8+TI2bdqEb7/9VqNepNJ07NgRLi4uGDhwIKZOnQoLCwvExcUhNTX1uc9dHCsrK8yZMwfZ2dkICgpCYmIivvzyS0REREjLTVu2bIn3338f/fv3x+HDh9GmTRvY2dkhPT0de/fuRf369fHBBx9ofW0zMzPMnDkTvXr1QmRkJIYMGYLc3FzMmjUL9+7dw/Tp05/r3g4fPoxBgwbh7bffRmpqKj777DN4eXlJ8xDq1KmDGjVqYPz48RBCwMXFBRs2bEBCQkKRc6l+pr766iv07dsXlpaWqF27NqpXr46pU6fis88+w7///ovXXnsNzs7OuHHjBg4ePAg7OztMmTJFq7gdHBzg6+uLdevWoW3btnBxcYGrqyucnZ0RFhaGnj17ok6dOnBwcMChQ4cQHx+Prl27lnhOVeJ44MABrZKNgoICLFu2DP7+/lKS9rROnTph/fr1uHXrltrqlWfJzMxE9erV8fbbb6Ndu3bw8fFBdnY2du3aha+++gr+/v6l3o8+41M5ceIEHjx4oPEKngrNoNNTyeCSkpJE3759RbVq1YSVlZWws7MTr7zyipg0aZK4efOmVC83N1d8/PHHwsfHR9jY2IiQkBCRlJT0zNUohw4dUruOagXFrVu31MqL2zQqPT1ddOvWTbi4uAgnJyfx3nvvicOHD2u0GiUxMVG0aNFC2NraCjc3NzFo0CBx9OjRIu/Nzc0VgwYNEm5ubkImk6mtInj6nlQyMzOFjY2NACC+//77Ytvz1q1bYsSIEcLPz09YWloKFxcX0bhxY/HZZ5+J7OzsYt+j4uvrK15//fUi5SEhISIkJESt7ODBgyI4OFjY2dkJLy8vER0dLRYvXlzsapTizoliNmkqbva96vtz4sQJERoaKmxsbISLi4v44IMPir2fH374QTRr1kzY2dkJGxsbUaNGDdGnTx9x+PBhtfupW7duiW3xtLVr14pmzZoJa2trYWdnJ9q2bSv27dunVqcsq1G2bt0qevfuLSpVqiRsbGxEx44dxfnz59XqJicni/bt2wsHBwfh7Ows3n77bZGSklLsapAJEyYIT09PYWZmVmRV1dq1a0VYWJhwdHQUcrlc+Pr6im7duolt27ZJdZ61iVpxP+vbtm0Tr7zyipDL5QKA6Nu3r3j06JEYOnSoaNCggXB0dBQ2Njaidu3aIjo6WqON01q3bi06duxYpLy4n5fC9wVAzJ8//5nnjY+PL7JaqaTVHrm5uWL27NkiIiJCVKtWTcjlcmFtbS38/f3Fxx9/LO7cuVPsdYr7GdBHfCoTJ04Urq6uxW6MSOpkQuhg8TIRUTkSFxeH/v3749ChQ8+ch1ARrV69Gj169MCVK1fUVjBRUQqFAjVr1kTPnj2fOU+L/sOlr0REBADo2rUrgoKCEBMTY+hQjN7PP/+M7OxsaedYKhmTDSIiAvB4Quz3338vLSumZ1MqlVi+fDkqVapk6FDKBQ6jEBERkV6xZ4OIiIj0iskGERER6RWTDSIiItKrCr+pl1KpRFpaGhwcHDTeQpmIiIgAIQTu378PT0/PIs/5KazCJxtpaWnw8fExdBhERETlVmpqaok7JFf4ZMPBwQHA44ZydHQ0cDRERETlR1ZWFnx8fKTP0mep8MmGaujE0dGRyQYREVEZlDYNgRNEiYiISK+YbBAREZFeMdkgIiIivaqwyUZsbCwCAgIQFBRk6FCIiIhMWoV/NkpWVhacnJyQmZnJCaJERERa0PQztML2bBAREdGLwWSDiIiI9IrJBhEREekVkw0iIiLSqwq/gygREZGpO3UtE1czHqBmFXvUrFLy1uL6wJ4NIiIiE7fiYAqG/nwUf564bpDrM9kgIiIycYbe44LJBhERUQVRyvPS9IbJBhEREekVkw0iIiITZ+i9wplsEBERVRAGGkVhskFERGT6DNu1wWSDiIioguAEUSIiItILztkwkNjYWAQEBCAoKMjQoRAREb0QMgN1bVTYZCMqKgrJyck4dOiQoUMhIiLSK/ZsEBERkUljskFERGTiBFejEBERkSljskFERFRBcOkrERER6QUniBIREdELITPQhuVMNoiIiEycgTs2mGwQERFVFJyzQURERHrBORtERERk0phsEBERmTjVpl4GGkVhskFERET6xWSDiIjI1D2Zs8EJokRERGSSmGwQERFVENzUi4iIiPSCm3oRERGRSWOyQUREZOLEk129OEGUiIiITBKTDSIiIhPHORsGEhsbi4CAAAQFBRk6FCIiIpNWYZONqKgoJCcn49ChQ4YOhYiISK+EtKkXl74SERGRCWKyQUREZOJUczb4IDYiIiIySUw2iIiISK+YbBAREZk4bupFREREJo3JBhERkYnjBFEiIiIyaUw2iIiITB039SIiIiJTxmSDiIjIxAkDP4qNyQYREVEFwaWvREREpBfCwM+YZ7JBRERUQXDpKxEREekFezaIiIjoxeDSVyIiIjJFTDaIiIhMnGrpK+dsEBERkUliskFERGTiOEGUiIiIXghu6kVERER6YeCODSYbREREpi5foQQAyAw0RZTJBhERkYk7m34fAKBQKg1yfSYbREREJq6Wuz0A4EGewiDXZ7JBRERk4pRPlqN4OFkb5PpMNoiIiEycavRExu3KX6zY2FgEBAQgKCjI0KEQERHpleJJz4Y5k40XKyoqCsnJyTh06JChQyEiItIr8STZMOM+G0RERKQPCuWTZMNA2QaTDSIiIhP3JNeAGYdRiIiISB9Uq1HMDfSpz2SDiIjIxKmSDa5GISIiIr14sls5V6MQERGRfpxJzwIAmHOCKBEREemD3OLxx72d3MIg12eyQUREZOKeTNmAu6PcINfXSbJx7949XZyGiIiI9KDgyX7l5WbOxowZM7Bq1Srpdffu3VG5cmV4eXnh+PHjOg2OiIiIno8Q4r99NsrLnI1FixbBx8cHAJCQkICEhARs3rwZERERGDdunM4DJCIiorJT7R4KABYGSja0nimSnp4uJRsbN25E9+7d0aFDB1SvXh3NmjXTeYBERERUdqqHsAHlqGfD2dkZqampAID4+Hi0a9cOwONuGoVCodvoiIiI6LmoHi8PGG7OhtY9G127dkXPnj1Rq1Yt3LlzBxEREQCApKQk1KxZU+cBEhERUdkVFMo2DLXPhtbJxrx581C9enWkpqZi5syZsLe3B/B4eGXYsGE6D5CIiIjKLq+gHCYblpaW+Oijj4qUjxo1ShfxEBERkQ49KpRsWBroSWwaJRvr16/X+ISdO3cuczBERESkW/lPkg0HA+0eCmiYbHTp0kXttUwmgyg0u7XwU+Q4SZSIiMh45D95CpulheE2DdfoykqlUvraunUrAgMDsXnzZty7dw+ZmZnYtGkTGjVqhPj4eH3HS0RERFrIfJgPALA0N8x8DaAMczZGjRqFb7/9Fq1atZLKwsPDYWtri/fffx9nzpzRaYBERERUdgf+vQMAeJhnuJEHrftULl68CCcnpyLlTk5OuHz5si5iIiIiIh1RTXVwd7Q2WAxaJxtBQUEYNWoU0tPTpbLr169j7NixaNq0qU6DIyIiouejmrMR5OdisBi0TjaWLFmCmzdvwtfXFzVr1kTNmjVRrVo1pKenY8mSJfqIkYiIiMqoQPF4QYeVgZa9AmWYs1GrVi0cP34c27Ztw9mzZyGEQEBAANq1a6e2KoWIiIgML+XuAwDlaIJoQUEBrK2tkZSUhA4dOqBDhw76iouIiIh04FhqBgAgXyFKqak/WvWpWFhYwNfXl3tpEBERlRPOtlYAgFru9gaLQesBnM8//xwTJkzA3bt39REPERER6ZCqR8PH2dZgMWg9Z+Prr7/GhQsX4OnpCV9fX9jZ2akdP3r0qM6CIyIioudT8GQ1ikV5mbMBFN26nIiIiIyXaulruVqNEh0drY84iIiISMdycgtw+c7j1SgW5SnZUDly5AjOnDkDmUyGgIAAvPLKK7qMi4iIiJ7TuRv3pf/3c7UroaZ+aZ1s3Lx5E++88w527dqFSpUqQQiBzMxMhIWFYeXKlXBzc9NHnERERKQl1YZeXpVs4GRjabA4tO5T+fDDD5GVlYXTp0/j7t27yMjIwKlTp5CVlYURI0boI0YiIiIqA9XkUDu5uUHj0LpnIz4+Htu2bYO/v79UFhAQgNjYWG7yRUREZETylY97NszNDDdfAyhDz4ZSqYSlZdGuGEtLSyiVSp0ERURERM/vYV4BAMNuVQ6UIdl49dVXMXLkSKSlpUll165dw+jRo9G2bVudBkdERERl8zBPgaE/P977ysKsnCUb//d//4f79++jevXqqFGjBmrWrAk/Pz/cv38fCxYs0EeMREREpKVr9x5K/x9e18OAkZRhzoaPjw+OHj2KhISEIk99JSIiIuNQ8GRqg6u9FYaE1DBoLFonGw8ePICtrS3at2+P9u3b6yMmIiIiek6qZa8WBp4cCpQh2ahUqRKaNGmC0NBQhIaGomXLlkWej0JERESGVSCtRDHsfA2gDHM2du/ejc6dO+Po0aPo1q0bnJ2d0bx5c4wfPx6bN2/WR4xERESkJcWTYRRDr0QBAJkQQpT1zQqFAocOHcK3336L5cuXQ6lUQqFQ6DI+vcvKyoKTkxMyMzPh6Oho6HCIiIh0Imr5Ufx5Mh013OywfWyoXq6h6WdomZ6NcvbsWezatQu7d+/Grl27kJ+fj06dOiEkJKTMARMREZFuFCiU+PNkOgDA0YDblKtonWx4eHggPz8fr776KkJDQ/Hpp5+ifv36+oiNiIiIykA1XwMA5rzd0ICRPKb1nA0PDw9kZ2cjJSUFKSkpuHr1KrKzs/URm8befPNNODs7o1u3bgaNg4iIyBjkK/7b0duzko0BI3lM62QjKSkJN27cwGeffYaCggJMnDgRbm5uaNasGcaPH6+PGEs1YsQILFu2zCDXJiIiMjaKQj0bht49FChDsgE8Xv7auXNnfPbZZ/j000/RvXt3HD16FLNmzdJ1fBoJCwuDg4ODQa5NRERkbAoPo5TLpa9//PEHRo4ciYYNG6JKlSr44IMPkJOTg3nz5uHEiRNaB7Bnzx506tQJnp6ekMlkWLt2bZE6CxcuhJ+fH6ytrdG4cWP89ddfWl+HiIioolD1bFiYySCTGT7Z0HqC6JAhQ9CmTRsMHjwYoaGhqFev3nMFkJOTg4YNG6J///546623ihxftWoVRo0ahYULF6Jly5ZYtGgRIiIikJycjGrVqml9vdzcXOTm5kqvs7Kynit+IiIiY5J69wHG/nYcgHH0agBlSDZu3ryp0wAiIiIQERHxzONz587FwIEDMWjQIADA/PnzsWXLFnzzzTeIiYnR+noxMTGYMmVKmeMlIiIyZr8dTsXBS3cBAG4OcgNH85jWwyihoaFYtmwZHj58WHrl55SXl4cjR46gQ4cOauUdOnRAYmJimc45YcIEZGZmSl+pqam6CJWIiMgoPMx/vLmmo7UFfhzQ1MDRPKZ1stG4cWN8/PHH8PDwwODBg3HgwAF9xAUAuH37NhQKBdzd3dXK3d3dcf36del1eHg43n77bWzatAne3t44dOjQM88pl8vh6Oio9kVERGQqVJND32vuixpu9gaO5jGtk405c+bg2rVrWLZsGW7duoU2bdogICAAs2fPxo0bN/QRY5HJLUIItbItW7bg1q1bePDgAa5evYqgoCC9xEFERGTslIUmhxqLMi19NTc3xxtvvIG1a9fi2rVr6NmzJyZOnAgfHx906dIFO3bs0Elwrq6uMDc3V+vFAB7PG3m6t4OIiIgKP+3V8I+WV3muSA4ePIhJkyZh9uzZqFKlCiZMmIAqVaqgU6dO+Oijj547OCsrKzRu3BgJCQlq5QkJCQgODn7u8xMREZkaadmrETztVaVMq1F++uknLF26FOfPn0enTp2wcuVKhIeHS0Mb3bt3R5cuXTB79uxSz5ednY0LFy5Iry9duoSkpCS4uLigWrVqGDNmDHr37o0mTZqgRYsW+O6775CSkoKhQ4dqGzoREZHJU/VsmBnB/hoqWicb3t7eqFGjBgYMGIB+/frBzc2tSJ2mTZtqPG/i8OHDCAsLk16PGTMGANC3b1/ExcWhR48euHPnDqZOnYr09HTUq1cPmzZtgq+vr7ahExERmaRH+Qr0WLQf/97OwaMnq1GMac6GTAghSq/2n7/++gutW7fWVzwvTGxsLGJjY6FQKPDPP/8gMzOTK1OIiKhcOpaSgTcX/rclhJkMWDagGVrVctXrdbOysuDk5FTqZ6jWyQYAFBQUYNeuXbh48SJ69uwJBwcHpKWlwdHREfb2xrHMRlOaNhQREZGxOnz5Lrp9ux9elWzw08CmcLC2fCEbemn6Gar1MMqVK1fw2muvISUlBbm5uWjfvj0cHBwwc+ZMPHr0CN9+++1zBU5ERETaUU0KtbY0w0tGsrdGYVqvRhk5ciSaNGmCjIwM2NjYSOVvvvkmtm/frtPgiIiIqHQKoVruajzzNArTumdj79692LdvH6ysrNTKfX19ce3aNZ0FRkRERJpRGOEKlMK07tlQKpVQKBRFyq9evQoHBwedBEVERESaUyiNu2dD62Sjffv2mD9/vvRaJpMhOzsb0dHR6Nixoy5jIyIiIg0ojHCL8sK0HkaZN28ewsLCEBAQgEePHqFnz544f/48XF1dsWLFCn3ESERERE9JvfsAl27nAABOXssEAJiZSrLh6emJpKQkrFy5EkeOHIFSqcTAgQPRq1cvtQmjREREpB+3s3Px6pxdyFeo715haUTPQylM62Tj559/xnvvvYf+/fujf//+asfGjRuHWbNm6Sw4fSq8qRcREVF5cj3zEfIVAhZmMrzs/ni+pIW5DP1aVjdsYM+g9aZelSpVws8//4zIyEi18tGjR2PlypVIT0/XaYD6xk29iIiovDlx9R46/98+eDpZI3FCW4PFoelnqNb9LStXrsR7772HPXv2SGUffvghfv31V+zcubNs0RIREZHGpKWuRjpH42laJxuvvfYavv32W3Tp0gWHDx/GsGHDsGbNGuzcuRN16tTRR4xERERUiNLIN/F6mtZzNgDgnXfeQUZGBlq1agU3Nzfs3r0bNWvW1HVsREREVAyF8vF/zY10E6+naZRsqB77/rQqVarglVdewcKFC6WyuXPn6iYyIiIiKpZqGKWc5BqaJRvHjh0rtrxGjRrIysqSjsvKy10TERGVYyY5jMKJn0RERMbD2J+F8rQyzdkgIiKiFyP17gNsOX1d6s0AgIs3H+8calI9G0RERGQYE9edwq5zt4o9Zmtl/oKjKZsKm2xwB1EiIioPMnLyAAAtXqqMqk7WUrmZmQzvBPkYKiytaL2DqKnhDqJERGTMIhf8hVPXsrC0fxDCalcxdDhqdLqDaKNGjZCRkQEAmDp1Kh48eKCbKImIiKhEynK2p0ZxNEo2zpw5g5ycx5NRpkyZguzsbL0GRURERI+pJoaWl5UnxdFozkZgYCD69++PVq1aQQiB2bNnw97evti6kyZN0mmAREREFdl/z0ExcCDPQaNkIy4uDtHR0di4cSNkMhk2b94MC4uib5XJZEw2iIiIdEjawMvUezZq166NlStXAgDMzMywfft2VKliXJNUiIiITNGTjo1y84TX4mi99FWpmqlCREREelfedgstTpn22bh48SLmz5+PM2fOQCaTwd/fHyNHjkSNGjV0HR8REVGF9t8EUQMH8hy0Tja2bNmCzp07IzAwEC1btoQQAomJiahbty42bNiA9u3b6yNOIiIik/EoX4G7TzbrKk3+k+fJl5etyYujdbIxfvx4jB49GtOnTy9S/sknnzDZICIiKsG9B3kImbULmQ/ztXpfeR5G0XohzZkzZzBw4MAi5QMGDEBycrJOgnoRYmNjERAQgKCgIEOHQkREFcil2zlSomFlYabRV0BVR9SsUvyWE+WB1j0bbm5uSEpKQq1atdTKk5KSytUKlaioKERFRUlbrRIREb0IqtUlvpVtsXtcmGGDeUG0TjYGDx6M999/H//++y+Cg4Mhk8mwd+9ezJgxA2PHjtVHjERERCZDmMCOoNrSOtmYOHEiHBwcMGfOHEyYMAEA4OnpicmTJ2PEiBE6D5CIiMiUqHo2KlCuoX2yIZPJMHr0aIwePRr3798HADg4OOg8MCIiIlNkCvtmaKtM+2yoMMkgIiLSjjCBfTO0VY4f60JERFT+SNuPV6CeDSYbREREL5ApPDJeW0w2iIiIXiAp2ahAn8Ba3Wp+fj7CwsLwzz//6CseIiIikyY4jFIyS0tLnDp1CrIK1EBERES6pFqNUpE+S7XuxOnTpw+WLFmij1iIiIhMnik8xVVbWi99zcvLw+LFi5GQkIAmTZrAzs5O7fjcuXN1FhwREZGx2Xv+NuISL0tJg7Zu3c8FAJhXoJ4NrZONU6dOoVGjRgBQZO5GeeoSio2NRWxsLBQKhaFDISKicmTBjvP4+9Ld5z6Pm4NcB9GUDzIhypiamQjVg9gyMzPh6Oho6HCIiMjIdYndh6TUe+jbwhd1Pcv2IE9zMxlCa7uhsn35Tjg0/Qwt8w6iFy5cwMWLF9GmTRvY2NhACFGuejaIiIjKQvUXeutabmgX4G7QWMoLrSeI3rlzB23btsXLL7+Mjh07Ij09HQAwaNAgPvWViIhMnqiA+2Q8L62bavTo0bC0tERKSgpsbW2l8h49eiA+Pl6nwRERERkb1cRQ9uZrTuthlK1bt2LLli3w9vZWK69VqxauXLmis8CIiIiMkVL5+L8VaVOu56V1z0ZOTo5aj4bK7du3IZeX74kuREREpVHN2WCqoTmtk402bdpg2bJl0muZTAalUolZs2YhLCxMp8EREREZG1EBH6T2vLQeRpk1axZCQ0Nx+PBh5OXl4eOPP8bp06dx9+5d7Nu3Tx8xEhERGY2KuAPo89K6ZyMgIAAnTpxA06ZN0b59e+Tk5KBr1644duwYatSooY8YiYiIjMaTR5twgqgWyrTPhoeHB6ZMmaLrWIiIiIyekFajGDiQcqRMyUZGRgaWLFmCM2fOQCaTwd/fH/3794eLi4uu4yMiIjIqFfER8c9L62GU3bt3w8/PD19//TUyMjJw9+5dfP311/Dz88Pu3bv1ESMREZHR4JwN7WndsxEVFYXu3bvjm2++gbm5OQBAoVBg2LBhiIqKwqlTp3QeJBERkbHgnA3tad2zcfHiRYwdO1ZKNADA3NwcY8aMwcWLF3UaHBERkbER4JwNbWnds9GoUSOcOXMGtWvXVis/c+YMAgMDdRUXERFRsbIe5ePIlYz/dtd6wR7mKQBwzoY2NEo2Tpw4If3/iBEjMHLkSFy4cAHNmzcHABw4cACxsbGYPn26fqIkIiJ64v1lh3Hg37uGDgMWnLShMZlQreEpgZmZGWQyGUqrKpPJoFAodBacPsXGxiI2NhYKhQL//PMPMjMz4ejoaOiwiIioFK1m7MDVjIeo4WYHO3mZFlU+t5dc7TD77YawMK/Yj37NysqCk5NTqZ+hGn2XLl26pLPAjEVUVBSioqKkhiIiovJB9XfvnO6BCPSpZNBYSDMaJRu+vr76joOIiEgj0qZaBo6DNFem/qdr165h3759uHnzJpSqZ+0+MWLECJ0ERkREVBzVgD4naJYfWicbS5cuxdChQ2FlZYXKlSurrTOWyWRMNoiISK+EtM+FYeMgzWmdbEyaNAmTJk3ChAkTYGZWsSfGEBHRi6csfV0DGRmts4UHDx7gnXfeYaJBREQGwWGU8kfrjGHgwIH47bff9BELERFRqTiMUv5oPYwSExODyMhIxMfHo379+rC0tFQ7PnfuXJ0FR0RE9DQhPQiN2UZ5oXWyMW3aNGzZskXarvzpCaJERET6pBpG4UdO+aF1sjF37lz88MMP6Nevnx7CISIiKhn32Sh/tJ6zIZfL0bJlS33EQkREVCo+4r380TrZGDlyJBYsWKCPWIiIiEol9Www1yg3tB5GOXjwIHbs2IGNGzeibt26RSaIrlmzRmfBERERPU2as2HQKEgbWicblSpVQteuXfURCxERUalUS1+5GqX8KNN25URERIbCYZTyh9uAEhFRufLfMAqzjfJC654NPz+/EmcA//vvv88VEBERUUmU7Nkod7RONkaNGqX2Oj8/H8eOHUN8fDzGjRunq7iIiIiKxe3Kyx+tk42RI0cWWx4bG4vDhw8/d0BERPR87j3Iw/1HBYYOQ28E99kod7RONp4lIiICEyZM4ARSIiIDOvDvHfRa/DcUStN/DDtTjfJDZ8nG77//DhcXF12dTu9iY2MRGxsLhUJh6FCIiHQmOS0LCqWAmQyQW5gbOhy9eaVaJXg4Whs6DNKQ1snGK6+8otZ1JYTA9evXcevWLSxcuFCnwelTVFQUoqKikJWVBScnJ0OHQ0SkE6r+jMgGnvj63VcMGguRitbJRpcuXdRem5mZwc3NDaGhoahTp46u4iIiojLgHhRkjLRONqKjo/URBxEREZkobupFRGSC2LFBxkTjng0zM7NSlxnJZDIUFJjucisiImPHZaFkjDRONv74449nHktMTMSCBQuksUIiIjIM8WSKKFMNMiYaJxtvvPFGkbKzZ89iwoQJ2LBhA3r16oUvvvhCp8EREZF2BJ+/TkaoTHM20tLSMHjwYDRo0AAFBQVISkrCjz/+iGrVquk6PiIi0gIfUkbGSKtkIzMzE5988glq1qyJ06dPY/v27diwYQPq1aunr/iIiEgLfG4IGSONh1FmzpyJGTNmwMPDAytWrCh2WIWIiAyLczbIGGmcbIwfPx42NjaoWbMmfvzxR/z444/F1luzZo3OgiMiIu2wZ4OMkcbJRp8+fbiUioionOCcDTImGicbcXFxegyDiIh0gduVkzHiDqJERCaEwyhkjJhsEBGZkP+2VmS2QcaDyQYRkQlhzwYZIyYbREQmhEtfyRgx2SAiMiHs2SBjxGSDiMiEcLtyMkZMNoiITAmXvpIRYrJBRGRC+NBXMkZMNoiITMh/czaYbpDxYLJBRGRCRKGdNoiMBZMNIiITwtUoZIyYbBARmRCuRiFjxGSDiMiEsGeDjBGTDSIiE8IdRMkYafyIeSKi5LQsXLv30NBhUAku384BwJ4NMi5MNohIIxduZqPj138ZOgzSkLkZO67JeFTYZCM2NhaxsbFQKBSGDoWoXEjPfNyjYW1pBv+qjgaOhkpiL7fAm694GToMIkmFTTaioqIQFRWFrKwsODk5GTocIqOnmnjo52qPP4a1NGwwRFSusJ+NiIiI9IrJBhFphM/cIKKyYrJBREREesVkg4g0IvjociIqIyYbRKQRPt6LiMqKyQYRaYU9G0SkLSYbRKQZ1TM3OEWUiLTEZIOIiIj0iskGEWlEesAXOzaISEtMNoiIiEivmGwQkUaENGeDiEg7TDaIiIhIr5hsEJFGhLRfOfs2iEg7TDaISCPc1IuIyorJBhFphf0aRKQtJhtEpBE+G4WIyorJBhEREekVkw0i0og0P9SgURBRecRkg4iIiPSKyQYRaUTa1IuTNohIS0w2iEhDXPxKRGXDZIOItMJ+DSLSFpMNItLIf8Moho2DiMofJhtERESkV0w2iEgj/y19ZdcGEWmHyQYRERHpFZMNItKI4K5eRFRGTDaIiIhIr5hsEJFGxJNZG+zYICJtMdkgIo0I7ulFRGXEZIOItMJ9NohIW0w2iEgjXPpKRGXFZIOIiIj0iskGEWlEPJm0wWEUItIWkw0iIiLSKyYbRKQV9mwQkbaYbBCRRrj0lYjKiskGEWmFq1GISFtMNohIIwLs2iCismGyQURa4ZwNItIWkw0i0gjnbBBRWTHZICIiIr1iskFEGlH1bMg4jkJEWjKJZGPjxo2oXbs2atWqhcWLFxs6HCIiIirEwtABPK+CggKMGTMGO3fuhKOjIxo1aoSuXbvCxcXF0KERmZT/HsRGRKSdct+zcfDgQdStWxdeXl5wcHBAx44dsWXLFkOHRWRyBGeIElEZGTzZ2LNnDzp16gRPT0/IZDKsXbu2SJ2FCxfCz88P1tbWaNy4Mf766y/pWFpaGry8vKTX3t7euHbt2osInahC4pQNItKWwYdRcnJy0LBhQ/Tv3x9vvfVWkeOrVq3CqFGjsHDhQrRs2RKLFi1CREQEkpOTUa1atWL/2jLkBLYbWY9wLCXDYNcn0peT1zIBcBiFiLRn8GQjIiICERERzzw+d+5cDBw4EIMGDQIAzJ8/H1u2bME333yDmJgYeHl5qfVkXL16Fc2aNXvm+XJzc5Gbmyu9zsrK0sFd/OdYyj0M/fmoTs9JZEzMzQzeIUpE5YzBk42S5OXl4ciRIxg/frxaeYcOHZCYmAgAaNq0KU6dOoVr167B0dERmzZtwqRJk555zpiYGEyZMkVvMVeytUQTX2e9nZ/IkCzNzdC/ZXVDh0FE5YxRJxu3b9+GQqGAu7u7Wrm7uzuuX78OALCwsMCcOXMQFhYGpVKJjz/+GJUrV37mOSdMmIAxY8ZIr7OysuDj46OzmJu/VBm/fxCss/MRERGVd0adbKg8PQdDCKFW1rlzZ3Tu3Fmjc8nlcsjlcp3GR0RERM9m1IOvrq6uMDc3l3oxVG7evFmkt4OIiIiMk1EnG1ZWVmjcuDESEhLUyhMSEhAczKEKIiKi8sDgwyjZ2dm4cOGC9PrSpUtISkqCi4sLqlWrhjFjxqB3795o0qQJWrRoge+++w4pKSkYOnSoAaMmIiIiTRk82Th8+DDCwsKk16rJm3379kVcXBx69OiBO3fuYOrUqUhPT0e9evWwadMm+Pr6GipkIiIi0oJMVNA9iGNjYxEbGwuFQoF//vkHmZmZcHR0NHRYRERE5UZWVhacnJxK/QytsMmGiqYNRUREROo0/Qw16gmiREREVP4x2SAiIiK9YrJBREREesVkg4iIiPTK4EtfDU01P1bXT38lIiIydarPztLWmlT4ZOP+/fsAoNOHsREREVUk9+/fh5OT0zOPV/ilr0qlEmlpaXBwcCjywLeyUj1JNjU1lctp9Yxt/WKxvV8stveLxfbWnhAC9+/fh6enJ8zMnj0zo8L3bJiZmcHb21sv53Z0dOQP7AvCtn6x2N4vFtv7xWJ7a6ekHg0VThAlIiIivWKyQURERHrFZEMP5HI5oqOjIZfLDR2KyWNbv1hs7xeL7f1isb31p8JPECUiIiL9Ys8GERER6RWTDSIiItIrJhtERESkV0w2iIiISK+YbOjYwoUL4efnB2trazRu3Bh//fWXoUMyqD179qBTp07w9PSETCbD2rVr1Y4LITB58mR4enrCxsYGoaGhOH36tFqd3NxcfPjhh3B1dYWdnR06d+6Mq1evqtXJyMhA79694eTkBCcnJ/Tu3Rv37t1Tq5OSkoJOnTrBzs4Orq6uGDFiBPLy8tTqnDx5EiEhIbCxsYGXlxemTp1a6p7/xiQmJgZBQUFwcHBAlSpV0KVLF5w7d06tDttcN7755hs0aNBA2gCqRYsW2Lx5s3Sc7axfMTExkMlkGDVqlFTGNjdignRm5cqVwtLSUnz//fciOTlZjBw5UtjZ2YkrV64YOjSD2bRpk/jss8/E6tWrBQDxxx9/qB2fPn26cHBwEKtXrxYnT54UPXr0EFWrVhVZWVlSnaFDhwovLy+RkJAgjh49KsLCwkTDhg1FQUGBVOe1114T9erVE4mJiSIxMVHUq1dPREZGSscLCgpEvXr1RFhYmDh69KhISEgQnp6eYvjw4VKdzMxM4e7uLt555x1x8uRJsXr1auHg4CBmz56tvwbSsfDwcLF06VJx6tQpkZSUJF5//XVRrVo1kZ2dLdVhm+vG+vXrxZ9//inOnTsnzp07Jz799FNhaWkpTp06JYRgO+vTwYMHRfXq1UWDBg3EyJEjpXK2ufFisqFDTZs2FUOHDlUrq1Onjhg/fryBIjIuTycbSqVSeHh4iOnTp0tljx49Ek5OTuLbb78VQghx7949YWlpKVauXCnVuXbtmjAzMxPx8fFCCCGSk5MFAHHgwAGpzv79+wUAcfbsWSHE46THzMxMXLt2TaqzYsUKIZfLRWZmphBCiIULFwonJyfx6NEjqU5MTIzw9PQUSqVShy3x4ty8eVMAELt37xZCsM31zdnZWSxevJjtrEf3798XtWrVEgkJCSIkJERKNtjmxo3DKDqSl5eHI0eOoEOHDmrlHTp0QGJiooGiMm6XLl3C9evX1dpMLpcjJCREarMjR44gPz9frY6npyfq1asn1dm/fz+cnJzQrFkzqU7z5s3h5OSkVqdevXrw9PSU6oSHhyM3NxdHjhyR6oSEhKht6BMeHo60tDRcvnxZ9w3wAmRmZgIAXFxcALDN9UWhUGDlypXIyclBixYt2M56FBUVhddffx3t2rVTK2ebGzcmGzpy+/ZtKBQKuLu7q5W7u7vj+vXrBorKuKnapaQ2u379OqysrODs7FxinSpVqhQ5f5UqVdTqPH0dZ2dnWFlZlVhH9bo8fg+FEBgzZgxatWqFevXqAWCb69rJkydhb28PuVyOoUOH4o8//kBAQADbWU9WrlyJo0ePIiYmpsgxtrlxq/BPfdW1px9TL4TQ2aPrTVVZ2uzpOsXV10Ud8WQyV3n8Hg4fPhwnTpzA3r17ixxjm+tG7dq1kZSUhHv37mH16tXo27cvdu/eLR1nO+tOamoqRo4cia1bt8La2vqZ9djmxok9Gzri6uoKc3PzIhnrzZs3i2S39JiHhweAoll+4Tbz8PBAXl4eMjIySqxz48aNIue/deuWWp2nr5ORkYH8/PwS69y8eRNA0b+WjN2HH36I9evXY+fOnfD29pbK2ea6ZWVlhZo1a6JJkyaIiYlBw4YN8dVXX7Gd9eDIkSO4efMmGjduDAsLC1hYWGD37t34+uuvYWFh8cxeA7a5cWCyoSNWVlZo3LgxEhIS1MoTEhIQHBxsoKiMm5+fHzw8PNTaLC8vD7t375barHHjxrC0tFSrk56ejlOnTkl1WrRogczMTBw8eFCq8/fffyMzM1OtzqlTp5Ceni7V2bp1K+RyORo3bizV2bNnj9ryta1bt8LT0xPVq1fXfQPogRACw4cPx5o1a7Bjxw74+fmpHWeb65cQArm5uWxnPWjbti1OnjyJpKQk6atJkybo1asXkpKS8NJLL7HNjdmLm4tq+lRLX5csWSKSk5PFqFGjhJ2dnbh8+bKhQzOY+/fvi2PHjoljx44JAGLu3Lni2LFj0nLg6dOnCycnJ7FmzRpx8uRJ8e677xa7VM3b21ts27ZNHD16VLz66qvFLlVr0KCB2L9/v9i/f7+oX79+sUvV2rZtK44ePSq2bdsmvL291Zaq3bt3T7i7u4t3331XnDx5UqxZs0Y4OjqWq6VqH3zwgXBychK7du0S6enp0teDBw+kOmxz3ZgwYYLYs2ePuHTpkjhx4oT49NNPhZmZmdi6dasQgu38IhRejSIE29yYMdnQsdjYWOHr6yusrKxEo0aNpCWHFdXOnTsFgCJfffv2FUI8Xq4WHR0tPDw8hFwuF23atBEnT55UO8fDhw/F8OHDhYuLi7CxsRGRkZEiJSVFrc6dO3dEr169hIODg3BwcBC9evUSGRkZanWuXLkiXn/9dWFjYyNcXFzE8OHD1ZalCSHEiRMnROvWrYVcLhceHh5i8uTJ5WqZWnFtDUAsXbpUqsM2140BAwZIv+tubm6ibdu2UqIhBNv5RXg62WCbGy8+Yp6IiIj0inM2iIiISK+YbBAREZFeMdkgIiIivWKyQURERHrFZIOIiIj0iskGERER6RWTDSIiItIrJhtERESkV0w2iKhEly9fhkwmQ1JSkqFDkZw9exbNmzeHtbU1AgMDDR0OEZWCyQaRkevXrx9kMhmmT5+uVr527doK+ahqAIiOjoadnR3OnTuH7du36+06u3btgkwmw71793R+7smTJzNRogqDyQZROWBtbY0ZM2YUeTR2eVb4aZjaunjxIlq1agVfX19UrlxZh1ERkT4w2SAqB9q1awcPDw/ExMQ8s05xfynPnz9f7XHW/fr1Q5cuXTBt2jS4u7ujUqVKmDJlCgoKCjBu3Di4uLjA29sbP/zwQ5Hznz17FsHBwbC2tkbdunWxa9cutePJycno2LEj7O3t4e7ujt69e+P27dvS8dDQUAwfPhxjxoyBq6sr2rdvX+x9KJVKTJ06Fd7e3pDL5QgMDER8fLx0XCaT4ciRI5g6dSpkMhkmT55c7HlU1xs+fDgqVaqEypUr4/PPP0fhx0H9/PPPaNKkCRwcHODh4YGePXvi5s2bAB4PH4WFhQEAnJ2dIZPJ0K9fPwCPHyU/c+ZMvPTSS7CxsUHDhg3x+++/S+dV9Yhs374dTZo0ga2tLYKDg3Hu3DkAQFxcHKZMmYLjx49DJpNBJpMhLi4OwOPvY7Vq1SCXy+Hp6YkRI0YUe39E5QmTDaJywNzcHNOmTcOCBQtw9erV5zrXjh07kJaWhj179mDu3LmYPHkyIiMj4ezsjL///htDhw7F0KFDkZqaqva+cePGYezYsTh27BiCg4PRuXNn3LlzBwCQnp6OkJAQBAYG4vDhw4iPj8eNGzfQvXt3tXP8+OOPsLCwwL59+7Bo0aJi4/vqq68wZ84czJ49GydOnEB4eDg6d+6M8+fPS9eqW7cuxo4di/T0dHz00UfPvFfV9f7++298/fXXmDdvHhYvXiwdz8vLwxdffIHjx49j7dq1uHTpkpRQ+Pj4YPXq1QCAc+fOIT09HV999RUA4PPPP8fSpUvxzTff4PTp0xg9ejTee+897N69W+36n332GebMmYPDhw/DwsICAwYMAAD06NEDY8eORd26dZGeno709HT06NEDv//+O+bNm4dFixbh/PnzWLt2LerXr1/i95OoXDDsQ2eJqDR9+/YVb7zxhhBCiObNm4sBAwYIIYT4448/ROFf4ejoaNGwYUO1986bN0/4+vqqncvX11coFAqprHbt2qJ169bS64KCAmFnZydWrFghhBDi0qVLAoCYPn26VCc/P194e3uLGTNmCCGEmDhxoujQoYPatVNTUwUAce7cOSHE48eBBwYGlnq/np6e4n//+59aWVBQkBg2bJj0umHDhiI6OrrE84SEhAh/f3+1R3p/8sknwt/f/5nvOXjwoAAg7t+/L4QQYufOnQKA2uPFs7OzhbW1tUhMTFR778CBA8W7776r9r5t27ZJx//8808BQDx8+FAIUfz3a86cOeLll18WeXl5Jd4bUXnDng2icmTGjBn48ccfkZycXOZz1K1bF2Zm//3qu7u7q/31bG5ujsqVK0vDCSotWrSQ/t/CwgJNmjTBmTNnAABHjhzBzp07YW9vL33VqVMHwOP5FSpNmjQpMbasrCykpaWhZcuWauUtW7aUrqWN5s2bq02ibdGiBc6fPw+FQgEAOHbsGN544w34+vrCwcEBoaGhAICUlJRnnjM5ORmPHj1C+/bt1e532bJlavcKAA0aNJD+v2rVqgBQpF0Le/vtt/Hw4UO89NJLGDx4MP744w8UFBRofd9ExsbC0AEQkebatGmD8PBwfPrpp1J3v4qZmZnafAQAyM/PL3IOS0tLtdcymazYMqVSWWo8qg9ypVKJTp06YcaMGUXqqD5kAcDOzq7UcxY+r4oQQucrb3JyctChQwd06NABP//8M9zc3JCSkoLw8PASJ6+q2uXPP/+El5eX2jG5XK72unC7Fm6rZ/Hx8cG5c+eQkJCAbdu2YdiwYZg1axZ2795d5HtEVJ4w2SAqZ6ZPn47AwEC8/PLLauVubm64fv262gezLvfGOHDgANq0aQMAKCgowJEjRzB8+HAAQKNGjbB69WpUr14dFhZl/2fF0dERnp6e2Lt3r3QtAEhMTETTpk3LFPPTr2vVqgVzc3OcPXsWt2/fxvTp0+Hj4wMAOHz4sFp9KysrAJB6QgAgICAAcrkcKSkpCAkJ0TqmwucufF4VGxsbdO7cGZ07d0ZUVBTq1KmDkydPolGjRmW+FpGhMdkgKmfq16+PXr16YcGCBWrloaGhuHXrFmbOnIlu3bohPj4emzdvhqOjo06uGxsbi1q1asHf3x/z5s1DRkaGNOExKioK33//Pd59912MGzcOrq6uuHDhAlauXInvv/8e5ubmGl9n3LhxiI6ORo0aNRAYGIilS5ciKSkJy5cv1zrm1NRUjBkzBkOGDMHRo0exYMECzJkzBwBQrVo1WFlZYcGCBRg6dChOnTqFL774Qu39vr6+kMlk2LhxIzp27AgbGxs4ODjgo48+wujRo6FUKtGqVStkZWUhMTER9vb26Nu3r0axVa9eHZcuXUJSUhK8vb3h4OCAFStWQKFQoFmzZrC1tcVPP/0EGxsb+Pr6an3vRMaEczaIyqEvvviiyJCJv78/Fi5ciNjYWDRs2BAHDx4scaWGtqZPn44ZM2agYcOG+Ouvv7Bu3Tq4uroCADw9PbFv3z4oFAqEh4ejXr16GDlyJJycnNTmh2hixIgRGDt2LMaOHYv69esjPj4e69evR61atbSOuU+fPnj48CGaNm2KqKgofPjhh3j//fcBPO4JiouLw2+//YaAgABMnz4ds2fPVnu/l5cXpkyZgvHjx8Pd3V3qyfniiy8wadIkxMTEwN/fH+Hh4diwYQP8/Pw0ju2tt97Ca6+9hrCwMLi5uWHFihWoVKkSvv/+e7Rs2RINGjTA9u3bsWHDBu4lQuWeTDz9LxYRkQkIDQ1FYGAg5s+fb+hQiCo89mwQERGRXjHZICIiIr3iMAoRERHpFXs2iIiISK+YbBAREZFeMdkgIiIivWKyQURERHrFZIOIiIj0iskGERER6RWTDSIiItIrJhtERESkV/8Pd1WdOun/AzoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "AI_PATSTAT = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/PATSTAT_Kwds_AI.parquet\")\n",
    "counts = sorted([el[0] for el in AI_PATSTAT.select(\"Kwd_count\").collect()])\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.figure(figsize=(6,4))\n",
    "plt.semilogy(np.linspace(0,AI_PATSTAT.count(), len(counts)), counts)\n",
    "plt.xlabel(\"Number of patents\")\n",
    "plt.ylabel(\"Number of keywords\")\n",
    "plt.title(\"Cumulative number of patents (PATSTAT)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3cf60cd5-b1bc-49ec-b00a-101961107999",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of publications in PATSTAT with 1+ Kwds 460820\n",
      "Number of publications in PATSTAT with 3+ Kwds 146022\n"
     ]
    }
   ],
   "source": [
    "AI_PATSTAT = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/PATSTAT_Kwds_AI.parquet\")\n",
    "print(\"Number of publications in PATSTAT with 1+ Kwds\", AI_PATSTAT.count())\n",
    "AI_PATSTAT = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/PATSTAT_Kwds3_AI.parquet\")\n",
    "print(\"Number of publications in PATSTAT with 3+ Kwds\", AI_PATSTAT.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ea48d23-acc4-42ce-bdea-487a192bc001",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAGHCAYAAAAZa5AYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABGsklEQVR4nO3deXxMV/8H8M9k3yOJSMSSKBqSWBMioZLUGhX1dEEpYmsRtVOq9j6WWko1tP1VLW2JPrWrhBSJ2gWxU1QJCUEjIYgkc35/MJcxWWZixkxmPu/XKy/mzJ0735mbZD4559xzZUIIASIiIiIdMdN3AURERGTcGDaIiIhIpxg2iIiISKcYNoiIiEinGDaIiIhIpxg2iIiISKcYNoiIiEinGDaIiIhIpxg2iIiISKcYNggnTpxAnz59UKNGDdjY2MDBwQGNGzfGl19+iX///Vff5ZVoypQpkMlkZXrs1q1bMWXKlCLv8/HxQXR0dNkLMyLR0dFwcHDQdxlqW7NmDfz9/WFrawuZTIbU1FR9lwQAkMlkxX6/acOMGTOwYcMGre935cqVcHd3x71795Ta8/Ly8M0336BFixZwcXGBlZUVqlSpgi5duiA5OVllP2fPnkV0dDSqV68OKysrVKxYER06dEB8fLzKtklJSZDJZNKXubk53N3dERUVhZSUFJXto6Ojlba3t7eHj48POnXqhGXLliEvL0/lMeHh4QgICFBqy83NxezZs9GgQQM4OTnB0dERNWvWVHlNO3bsgIODA65fv672+2jyBJm077//XlhYWAh/f38RGxsrdu3aJbZv3y5mzJghatSoITp37qzvEks0efJkUdZv45iYmGIfe/ToUXHx4sWXKc1o9O7dW9jb2+u7DLVkZmYKS0tLERUVJZKSksT+/ftFbm6uvssSQgixf/9+kZaWprP929vbi969e2t1n7m5uaJKlSpizpw5Su23bt0SgYGBwtLSUnz88cdiw4YNYvfu3WL16tWiW7duwtzcXKSmpkrbr127VlhbW4u6deuK77//XiQnJ4tff/1VREZGCgBizJgxSvvftWuXACBmzJgh9u/fL3bv3i0WLlwoXF1dhZ2dnfjrr7+Utu/du7ewtbUV+/fvF/v37xc7d+4UK1askGrx9/dXee/DwsKEv7+/dLugoECEhoYKR0dHMW3aNJGQkCASEhLEokWLRNu2bcX06dOVHh8RESF69er1Uu+vKWHYMGH79u0T5ubmon379uLRo0cq9+fl5YmNGzfqoTL16Sps0DOvKmxoIxTs2bNHABBr1qzRQkUlM5QQo6CLsLF48WJhY2MjsrKylNojIyOFhYWF2LFjR5GPO3TokLhy5YoQQoiLFy8KOzs7ERQUJO7fv6+y7cCBAwUAsXr1aqlNETb+97//KW27YsUKAUBMmjRJqb2k79Ft27YJS0tLERwcrNT+YtjYuXOnACB+/PHHIvdTWFiodPu3334T5ubm4urVq0VuT8r4m9aEdezYUVhYWKj9wwJATJ48WaXd29tb6ZfcsmXLBACxY8cO0b9/f+Hq6iocHR1Fz549xf3790VGRoZ4//33hbOzs/D09BSjRo0Sjx8/lh6v+EWza9cupee5fPmyACCWLVsmtRUVNuLi4kSbNm2Ep6ensLGxEXXq1BGffvqp0i+63r17CwAqX5cvX1Z5TYq/lj///HOV13727FkBQCxcuFBqy8jIEB999JGoUqWKsLS0FD4+PmLKlCkiPz+/lHf4yfO+9dZbIj4+XjRq1EjY2NgIX19fsXTpUqXtigtZivde8Tqe3+fmzZtFw4YNpfdk8+bN0mPq1Kkj7OzsRJMmTcThw4eV9qn4RX7q1Cnx5ptvCjs7O1GxYkURExOj8oErl8tFbGysaNCggbCxsREVKlQQ7777rrh06ZLSdopf9MnJySIkJETY2tqKrl27lvjebNy4UTRr1kzY2toKBwcH0bp1a7Fv3z6lOl88nmFhYcXuT/Febd++XURHRwsXFxdhZ2cnOnbsqFG9V65cET169BDu7u7CyspK1KlTR8ydO1flw6monx91v1cePXokpk6dKurUqSOsra2Fq6urCA8PF3v37pX2Xdxrz83NFaNGjRI+Pj7C2tpauLi4iMDAQLFq1aoS328hhKhXr554//33ldpSUlIEAPHxxx+X+nghnoX6/fv3F3l/bm6uqFChgggICJDaigsbp0+fLvK5SwvEgwcPFgBEcnKy1PZi2Pjf//4nAIj4+Hi1XldeXp5wdnYWEydOVGt7U8c5GyaqsLAQO3fuRGBgIKpVq6aT5+jfvz+cnZ0RFxeHzz//HKtWrcKAAQPw1ltvoUGDBvjtt9/Qu3dvzJs3D4sWLdLa8164cAEdOnTA0qVLkZCQgOHDh+PXX39FVFSUtM3EiRPx3nvvAQD2798vfVWuXFllf+7u7ujYsSNWrFgBuVyudN+yZctgZWWFHj16AABu3LiBpk2bYtu2bZg0aRLi4+PRr18/zJw5EwMGDFCr/uPHj2PUqFEYMWIENm7ciPr166Nfv37YvXt3Wd8SHD9+HOPHj8enn36KdevWwdnZGe+88w4mT56MH374ATNmzMAvv/yC7OxsdOzYEQ8fPlR6fH5+Pjp06IBWrVphw4YNGDJkCL777jt07dpVabuPP/4Yw4cPR+vWrbFhwwYsXrwYp0+fRmhoKG7evKm0bUZGBj788EN0794dW7duxeDBg4utf9WqVXj77bfh5OSE1atXY+nSpcjKykJ4eDj27NkD4MkxjY2NBfBk/sL+/fuxePHiUt+bfv36wczMDKtWrcKCBQtw6NAhhIeH4+7du6XWe+vWLYSGhmL79u2YPn06Nm3ahNatW2P06NEYMmRIic+r7vdKQUEBIiMjMX36dHTs2BHr16/H8uXLERoaiqtXrwJ48j1sa2uLDh06SN/Litc+cuRILFmyBEOHDkVCQgJ++uknvP/++7hz506J9V27dg0nT55ERESEUvv27dsBAJ07dy71vQWAxMREeHh4oFmzZkXeb2dnh7Zt2+LUqVO4ceNGifu6fPkyAOD1119X67kVOnXqBAAl/gwFBQXB0tISw4YNwy+//IKMjIwS92llZYXQ0FD8/vvvGtVisvSddkg/bty4IQCIbt26qf0YaNiz8cknnyht17lzZwFAzJ8/X6m9YcOGonHjxtLtl+3ZeJ5cLhf5+fkiOTlZABDHjx+X7itpGOXF17Rp0ybpr2CFgoIC4eXlJd59912p7eOPPxYODg5SF7LC3LlzBQBx+vTpYmtVPK+NjY3S4x8+fChcXV2V/prTtGfD1tZWXLt2TWpLTU0VAETlypWVeic2bNggAIhNmzZJbYoeg+d7b4QQ4r///a8AIPbs2SOEeDInAYCYN2+e0nZpaWnC1tZWjB07VmoLCwuTer9KU1hYKLy8vES9evWUegvu3bsnKlWqJEJDQ6W24v4iLorivfrPf/6j1L53714BQHzxxRel1jtu3DgBQBw8eFCpfdCgQUImk4nz589LbS/+/Kj7vbJy5UoBQPzf//1fia+nuGGUgICAMs29WrNmjQAgDhw4oNSuGPY4d+6cWvuxsbERzZo1K3GbTz/9VOl9VBzHNWvWiPz8fPHgwQOxd+9e4evrK/z8/FSGdUrr2VD0QA4aNEhqe7FnQwghli5dKhwcHKTeocqVK4tevXqJ3bt3F7nfCRMmCDMzsyKHh0gZezZIZzp27Kh0u27dugCAt956S6X9ypUrWnvev//+G927d4enpyfMzc1haWmJsLAwAE9mxJdFZGQkPD09sWzZMqlt27ZtSE9PR9++faW2LVu2ICIiAl5eXigoKJC+IiMjAaDIWfovatiwIapXry7dtrGxweuvv/5S71HDhg1RpUoV6bbiWISHh8POzk6lvajnUvTeKHTv3h0AsGvXLgBPXrtMJsOHH36o9No9PT3RoEEDJCUlKT3excUFb775Zqm1nz9/Hunp6ejZsyfMzJ79ynJwcMC7776LAwcO4MGDB6Xupzgvvq7Q0FB4e3tLr6ukenfu3Ak/Pz80bdpUqT06OhpCCOzcubPY51X3eyU+Ph42NjZK32eaaNq0KeLj4zFu3DgkJSWp9FoVJz09HQBQqVKlMj2vJoQQAKByZlnXrl1haWkJOzs7NG/eHDk5Ofj9999RoUKFMu2/NH379sW1a9ewatUqDB06FNWqVcPPP/+MsLAwzJkzR2X7SpUqQS6Xl9ojQzz11WRVrFgRdnZ2UrekLri6uirdtrKyKrb90aNHWnnO+/fv44033sDBgwfxxRdfICkpCYcPH8a6desAQO1ftC+ysLBAz549sX79eql7ffny5ahcuTLatWsnbXfz5k1s3rwZlpaWSl/+/v4AgNu3b5f6XG5ubipt1tbWZa4d0OxYAFA5HhYWFip1eXp6AoDUHX/z5k0IIeDh4aHy+g8cOKDy2osasiqKYv9Fbe/l5QW5XI6srCy19lUUxet4se3FYYainv/OnTvF1qW4vzjqfq/cunULXl5eSkFLE19//TU+/fRTbNiwAREREXB1dUXnzp1x4cKFEh+n+H6zsbFRalcEYXV/d1SvXr3Ubf/55x8AUBnSnT17Ng4fPozk5GRMmDABN2/eROfOnYs8lbUkivCsOC4lcXZ2xgcffICFCxfi4MGDOHHiBDw8PDBhwgSVoTXFe/MyP5umwkLfBZB+mJubo1WrVoiPj8e1a9dQtWrVUh9jbW1d5A95aWO/mlL8AL/4XOp8UO/cuRPp6elISkqSejMAqPySKIs+ffpgzpw5iIuLQ9euXbFp0yYMHz4c5ubm0jYVK1ZE/fr18d///rfIfajzy04dz79H1tbWUrs671FZFBQU4M6dO0qBQ/HXnKKtYsWKkMlk+PPPP5VqUnixTd31URT7L2oMPT09HWZmZnBxcVHvhRShqL9Kb9y4gVq1aim1FVWvm5tbsXUBT96T4qj7veLu7o49e/ZALpeXKXDY29tj6tSpmDp1Km7evCn1ckRFReHcuXMl1gcA//77r1KgateuHT777DNs2LAB7du3L/X527Rpg9jYWBw4cKDIeRsPHjxAYmIiAgICVILfa6+9hqCgIABAy5YtYWtri88//xyLFi3C6NGj1Xr9ALBp0yYAT3ryNOXv749u3bphwYIF+Ouvv5R6sRTrEJV0nOkJ9myYsPHjx0MIgQEDBuDx48cq9+fn52Pz5s3SbR8fH5w4cUJpm507d+L+/ftarcvHxwcAVJ5L8QujJIoPhBc/2L777juVbRXbqPtXSd26dREcHIxly5Zh1apVyMvLQ58+fZS26dixI06dOoWaNWsiKChI5UtbYaO49+j546Vtv/zyi9LtVatWAXj2C7xjx44QQuD69etFvvZ69eqV6Xl9fX1RpUoVrFq1Sqk7PDc3F2vXrkVISIjSUJCmXnxd+/btw5UrV9T6YGrVqhXOnDmDo0ePKrWvXLkSMplMZXLl89T9XomMjMSjR4+wfPnyEmtRp/fLw8MD0dHR+OCDD3D+/PkSh5/q1KkDALh06ZJSe+PGjREZGYmlS5cWO0yUkpIiTV4dMWIEbG1t8cknnyA3N1dl29GjRyMrKwuff/55ibUDwNixY1GrVi3MmjVLZZGx4iQmJuKHH35AaGgoWrRoUex2d+7cKfL3IAAplL348/v333/Dzc0NHh4eatViytizYcJCQkKwZMkSDB48GIGBgRg0aBD8/f2Rn5+PY8eO4fvvv0dAQIB0FkfPnj0xceJETJo0CWFhYThz5gy++eYbODs7a7UuT09PtG7dGjNnzoSLiwu8vb2xY8cOaSikJKGhoXBxccHAgQMxefJkWFpa4pdffsHx48dVtlV8+M2ePRuRkZEwNzdH/fr1paGEovTt2xcff/wx0tPTERoaCl9fX6X7p02bhsTERISGhmLo0KHw9fXFo0eP8M8//2Dr1q349ttv1epFKk2HDh3g6uqKfv36Ydq0abCwsMDy5cuRlpb20vsuipWVFebNm4f79++jSZMm2LdvH7744gtERkZKv8CbN2+Ojz76CH369EFKSgpatmwJe3t7ZGRkYM+ePahXrx4GDRqk8XObmZnhyy+/RI8ePdCxY0d8/PHHyMvLw5w5c3D37l3MmjXrpV5bSkoK+vfvj/fffx9paWmYMGECqlSpUuLZMQojRozAypUr8dZbb2HatGnw9vbG77//jsWLF2PQoEElnjWh7vfKBx98gGXLlmHgwIE4f/48IiIiIJfLcfDgQdStWxfdunUD8OT7OSkpCZs3b0blypXh6OgIX19fBAcHo2PHjqhfvz5cXFxw9uxZ/PTTT6WGtODgYNja2uLAgQPS2RwKK1euRPv27REZGYm+ffsiMjISLi4uyMjIwObNm7F69WocOXIE1atXR82aNfHTTz+hR48eaNKkCUaOHAlfX1/cvHkTP/74I+Lj4zF69GiVM5uKYmlpiRkzZqBLly5YuHChUkCRy+U4cOAAgCc9flevXkV8fDx+/fVX1K1bF7/++muJ+961axeGDRuGHj16IDQ0FG5ubsjMzMTq1auRkJCAXr16qfzsHjhwAGFhYWVexdik6HFyKhmI1NRU0bt3b1G9enVhZWUl7O3tRaNGjcSkSZNEZmamtF1eXp4YO3asqFatmrC1tRVhYWEiNTW12LNRXlyvQXEGxa1bt5Tai5pJnpGRId577z3h6uoqnJ2dxYcffiid31/a2Sj79u0TISEhws7OTri7u4v+/fuLo0ePqjw2Ly9P9O/fX7i7uwuZTFbsOhvPy87OFra2tiWeHXDr1i0xdOhQUaNGDWFpaSlcXV1FYGCgmDBhQqmz1hVrYrwoLCxMZc2IQ4cOidDQUGFvby+qVKkiJk+eLH744Ydi19l4EQARExOj1KY44+f5FSMVx+fEiRMiPDxc2NraCldXVzFo0KAiX8+PP/4ogoODhb29vbC1tRU1a9YUvXr1EikpKUqv58UzAUqzYcMGERwcLGxsbIS9vb1o1aqVtM6EQlnORtm+fbvo2bOnqFChgrC1tRUdOnQQFy5cUNq2pHqvXLkiunfvLtzc3ISlpaXw9fUVc+bMKXKdjSlTpii1qfu98vDhQzFp0iRRu3ZtYWVlJdzc3MSbb76ptM5IamqqaN68ubCzs1NaZ2PcuHEiKChIuLi4CGtra/Haa6+JESNGiNu3b5f6HvXs2VP4+fkVed/Dhw/F119/LUJCQoSTk5OwsLAQXl5e4p133hG///67yvanT58WvXv3FlWrVpVea/v27YvctrTjGBwcLFxcXMTdu3eFEKprrNja2orq1auLqKgo8eOPP4q8vDyVfbx4TNPS0sTnn38umjdvLjw9PYWFhYVwdHQUwcHBYtGiRaKgoEDp8RcvXhQAxNq1a4t/A0kiE0LNabpEREZk+fLl6NOnDw4fPizNC9CV7OxsVKhQAYsWLSp1/Q1DkpKSgiZNmuDAgQMIDg7WdzkGZeLEiVi5ciUuXboECwsOEpSGczaIiHTowIED0gXYQkJC9FuMhoKCgtClSxdMnz5d36UYlLt37yI2NhYzZsxg0FAT3yUiIh3q3r07CgsLMW/ePAQGBuq7HI3NmzcPS5cuxb179+Do6KjvcgzC5cuXMX78eGmtGSodh1GIiIhIpziMQkRERDrFsEFEREQ6xbBBREREOmXyE0TlcjnS09Ph6OjIhVmIiIg0IITAvXv3Sr1+j8mHjfT0dJWL/xAREZH60tLSSlwd2eTDhuJUrrS0NDg5Oem5GiIiovIjJycH1apVK/W0aJMPG4qhEycnJ4YNIiKiMihtGgIniBIREZFOMWwQERGRTjFsEBERkU4xbBAREZFOMWwQERGRTjFsEBERkU4xbBAREZFOMWwQERGRTjFsEBERkU6Z/AqiRERExu7U9Wxcy3qAWpUcUKtSyUuL6wJ7NoiIiIxc3OGrGPjzUWw5kaGX52fYICIiMnJy8eRfs1KuYaIrDBtERERGTognacNMP1mDYYOIiMjYyeVP/i3t6qy6wrBBRERk5ORSzwbDBhEREemAYs6GnrIGwwYREZGx45wNIiIi0ikOoxAREZFOPRtGYdggIiIiHXiaNTiMQkRERLrBYRQiIiLSKU4QJSIiIp3iol5ERESkUxxGISIiIp3iol5ERESkU5yzQURERDqlGEbhnA0iIiLSCcUwCudsEBERkU7IOYxCREREuiTYs0FERES69GzOhn6en2GDiIjIyHGdDSIiItIpwXU2iIiISJc4Z4OIiIh0imejEBERkU5xUS8iIiLSqaNX7wLgMAoRERHpSEUHKwCAnkZRGDaIiIiMnWK58upudnp5foYNIiIiI5dfIAcAWJrr52OfYYOIiMjIPS58EjasLBg2iIiISMuEEFLYsDTnBFEiIiLSskK5kBb1suIwChEREWlbfqGQ/s9hFCIiItK63McF0v85QZSIiIi07kb2I+n/Fnpar5xhg4iIyIgVPF1ko6KDFZcrJyIiIu3Lf3omiqONpd5qYNggIiIyYs8W9NLXYuUMG0REREbt2Rob+vvIZ9ggIiIyYgVPT321YNggIiIiXVDM2bDiMAoRERHpQvbDfAAcRiEiIiId+W733wAAPZ31CoBhg4iIyKg52T455bWWu4PeamDYICIiMmIFT+dsRNSppLcaGDaIiIiM2LMJopyzQURERDrAU1+15D//+Q9cXFzw3nvv6bsUIiIig6JY1MuCp76+nKFDh2LlypX6LoOIiMjgKHo2OIzykiIiIuDo6KjvMoiIiAxOgZw9G9i9ezeioqLg5eUFmUyGDRs2qGyzePFi1KhRAzY2NggMDMSff/756gslIiIqhx4/vRCbhZkJ92zk5uaiQYMG+Oabb4q8f82aNRg+fDgmTJiAY8eO4Y033kBkZCSuXr1apufLy8tDTk6O0hcREZExupnzCDmPCgCY+DBKZGQkvvjiC7zzzjtF3j9//nz069cP/fv3R926dbFgwQJUq1YNS5YsKdPzzZw5E87OztJXtWrVXqZ8IiIig7X34m3p/+6O1nqrQ+9hoySPHz/GkSNH0LZtW6X2tm3bYt++fWXa5/jx45GdnS19paWlaaNUIiIig6NYYyPQ2wW2VuZ6q8NCb8+shtu3b6OwsBAeHh5K7R4eHrhx44Z0u127djh69Chyc3NRtWpVrF+/Hk2aNClyn9bW1rC21l+6IyIielUK5E/ORHGzt9JrHQYdNhRkL1w9Rgih1LZt27ZXXRIREZHBU5z2qs8rvgIGPoxSsWJFmJubK/ViAEBmZqZKbwcREREpyzeABb0AAw8bVlZWCAwMRGJiolJ7YmIiQkND9VQVERFR+aAYRtHnaa+AAQyj3L9/HxcvXpRuX758GampqXB1dUX16tUxcuRI9OzZE0FBQQgJCcH333+Pq1evYuDAgXqsmoiIyPAprvhqYabfng29h42UlBRERERIt0eOHAkA6N27N5YvX46uXbvizp07mDZtGjIyMhAQEICtW7fC29tbXyUTERGVC/nSRdhMPGyEh4dDCFHiNoMHD8bgwYNfUUVERETGQbFUOSeIEhERkdYVygV+TbkGQP/DKAwbRERERmjPxdu4dS8PAGBvrd+BDJMNG7GxsfDz8yt28S8iIqLy7N/cPOn/3YOr67ESEw4bMTExOHPmDA4fPqzvUoiIiLROsaBXhK87PJxs9FqLyYYNIiIiY1b4dI0Ncz2vsQEwbBARERmlfGlBL/1ODgUYNoiIiIxS4dMFvcz1vMYGwLBBRERklBRLlVsaS8/G3bt3tbEbIiIi0pKC8jxnY/bs2VizZo10u0uXLnBzc0OVKlVw/PhxrRZHREREZVNYnudsfPfdd6hWrRqAJ1dfTUxMRHx8PCIjIzFmzBitF0hERESaKzCQ66IAZbg2SkZGhhQ2tmzZgi5duqBt27bw8fFBcHCw1gskIiIizSmui1IuezZcXFyQlpYGAEhISEDr1q0BAEIIFBYWarc6IiIi0sicbefQ7qvd+OXgVQCGMWdD456Nd955B927d0ft2rVx584dREZGAgBSU1NRq1YtrReoK7GxsYiNjWVAIiIioyGEwOKkS3j+Yurebnb6K+gpjcPGV199BR8fH6SlpeHLL7+Eg4MDgCfDK+XpMvAxMTGIiYlBTk4OnJ2d9V0OERHRSyuQCylofN8zEB5ONqhfVf+fcRqHDUtLS4wePVqlffjw4dqoh4iIiMpIcQYKAITUdIOjjaUeq3lGrbCxadMmtXfYqVOnMhdDREREZfd82LAwgLkaCmqFjc6dOyvdlslkEM8NCMlkz2a6cg4EERGRfhQ8FzbMDeAsFAW1Yo9cLpe+tm/fjoYNGyI+Ph53795FdnY2tm7disaNGyMhIUHX9RIREVExlHs2DCdsaDxnY/jw4fj222/RokULqa1du3aws7PDRx99hLNnz2q1QCIiIlKPImzIZICZAYUNjQd0Ll26VOTZG87Ozvjnn3+0URMRERGVgSEtUf48jcNGkyZNMHz4cGRkZEhtN27cwKhRo9C0aVOtFkdERETqU6waaiYr52Fj6dKlyMzMhLe3N2rVqoVatWqhevXqyMjIwNKlS3VRIxEREanBUHs2NJ6zUbt2bRw/fhx//PEHzp07ByEE/Pz80Lp1a6WzUoiIiOjVenZZecP6PNYobBQUFMDGxgapqalo27Yt2rZtq6u6iIiISENyRc+GueGssQFoGDYsLCzg7e3NtTSIiIgMxKP8QizccQE3cx4h52E+AMPr2dA4+nz++ecYP348/v33X13U88rExsbCz88PTZo00XcpREREZbb7r1tYknQJ645exx9nMwEArnZWeq5KmUw8vxSoGho1aoSLFy8iPz8f3t7esLe3V7r/6NGjWi1Q1xQXYsvOzoaTk5O+yyEiItLIxtTrGBaXCh83O3QPrg4ZZAj3dUdtD0edP7e6n6EaTxB9celyIiIi0p+Cwid9BtXd7PFRy5p6rqZoGoeNyZMn66IOIiIiKoPCpwMU5oY1TUOJxmFD4ciRIzh79ixkMhn8/PzQqFEjbdZFREREapAb6Omuz9M4bGRmZqJbt25ISkpChQoVIIRAdnY2IiIiEBcXB3d3d13USUREREUw1LU1nqfx2SiffPIJcnJycPr0afz777/IysrCqVOnkJOTg6FDh+qiRiIiIiqGXBh+2NC4ZyMhIQF//PEH6tatK7X5+fkhNjaWi3wRERG9Yoolyg3teijP07hnQy6Xw9LSUqXd0tIS8qcXgCEiIqJXo9AYh1HefPNNDBs2DOnp6VLb9evXMWLECLRq1UqrxREREVHJjDJsfPPNN7h37x58fHxQs2ZN1KpVCzVq1MC9e/ewaNEiXdRIRERExXh26qvhhg2N52xUq1YNR48eRWJiospVX4mIiOjVKiw0/J4NjcPGgwcPYGdnhzZt2qBNmza6qImIiIjUVGiMZ6NUqFABQUFBCA8PR3h4OJo3b65yfRQiIiJ6NYxyUa/k5GQkJycjKSkJ33zzDR49eoTGjRsjPDwcYWFhiIyM1EWdREREJu3Y1Sycu3FPpf3k9WwAhn3qq8ZXfX1eYWEhDh8+jG+//Ra//PIL5HI5CgsLtVmfzvGqr0REZOiyH+Qj6L+JyC8s/iN7YFhNjIus8wqr0uFVXwHg3LlzSEpKkno48vPzERUVhbCwsDIX/KrFxsYiNja23IUjIiIyPdkP85FfKGAmA1rV9VC5397KHB80raaHytSjcc+Gp6cn8vPz8eabbyI8PBwtW7ZEvXr1dFWfzrFng4iIDN3l27mImJsEB2sLnJraTt/lSNT9DNV4nQ1PT0/cv38fV69exdWrV3Ht2jXcv3//pYolIiKi4j1bklzPhZSRxmEjNTUVN2/exIQJE1BQUICJEyfC3d0dwcHBGDdunC5qJCIiMmnl4WJrJSnTnI0KFSqgU6dOaNGiBZo3b46NGzdi1apVSElJwaxZs7RdIxERkUkzubCxfv16JCUlISkpCadPn4abmxveeOMNfPXVV4iIiNBFjURERCatPFzZtSQah42PP/4YLVu2xIABAxAeHo6AgABd1EVERERPKS6qbjI9G5mZmbqog4iIiIqhWJK8vPZsaDxBNDw8HCtXrsTDhw91UQ8RERG9QDFnw0zjT23DoHHZgYGBGDt2LDw9PTFgwAAcOHBAF3URERHRU9L1T0ylZ2PevHm4fv06Vq5ciVu3bqFly5bw8/PD3LlzcfPmTV3USEREZNKkCaLldM5GmTpkzM3N8fbbb2PDhg24fv06unfvjokTJ6JatWro3Lkzdu7cqe06iYiITJZ0GXlT6dl43qFDhzBp0iTMnTsXlSpVwvjx41GpUiVERUVh9OjR2qqRiIjIpCkuLFJeJ4iW6WyUn376CcuWLcOFCxcQFRWFuLg4tGvXDrKnb0KXLl3QuXNnzJ07V+sFExERmZryPoyicdioWrUqatasib59+yI6Ohru7u4q2zRt2hRNmjTRSoFERESmThpGKadno2gcNnbs2IE33nijxG2cnJywa9euMhdFREREz5T3s1E0DhtvvPEGCgoKkJSUhEuXLqF79+5wdHREeno6nJyc4ODgoIs6iYiIjFJ+oRy7zmUi+2F+sducTs8BYELDKFeuXEH79u1x9epV5OXloU2bNnB0dMSXX36JR48e4dtvv9VFnUREREZpw7HrGPPbCbW2tbYon+MoGoeNYcOGISgoCMePH4ebm5vU/p///Af9+/fXanG6FBsbi9jYWBQWFuq7FCIiMmGZ9/IAAJWdbVDH07HY7czNZIgOrfGqytIqjcPGnj17sHfvXlhZWSm1e3t74/r161orTNdiYmIQExODnJwcODs767scIiIyUeLp5M+w190x6936eq5GNzTuj5HL5UX2Bly7dg2OjsUnMiIiIlL1dO6ntHyEMdI4bLRp0wYLFiyQbstkMty/fx+TJ09Ghw4dtFkbERGR0ZOX89Na1aHxMMpXX32FiIgI+Pn54dGjR+jevTsuXLiAihUrYvXq1bqokYiIyGgpTmstr6uDqkPjsOHl5YXU1FTExcXhyJEjkMvl6NevH3r06AFbW1td1EhERGS05OV8KXJ1aBw2fv75Z3z44Yfo06cP+vTpo3TfmDFjMGfOHK0VR0REZOwUwyjGHDY0HiEaMmQItmzZotI+YsQI/Pzzz1opioiIyFQUSmFDz4XokMZhIy4uDh9++CF2794ttX3yySf49ddfuUQ5ERGRhqQruhpx2tA4bLRv3x7ffvstOnfujJSUFAwePBjr1q3Drl27UKdOHV3USEREZLQUE0SNeBRF8zkbANCtWzdkZWWhRYsWcHd3R3JyMmrVqqXt2oiIiIyeYoJoeb3ImjrUChsjR44ssr1SpUpo1KgRFi9eLLXNnz9fO5URERGZAFOYIKpW2Dh27FiR7TVr1kROTo50vzGvfkZERKQLchOYIKpW2ODETyIiIt2QwoYRpw0jXhyViIjI8BXKn/xrzMMoDBtERER6JExgGIVhg4iISI8UwyjGPO+RYYOIiEiPpFNfjbhrQ62w0bhxY2RlZQEApk2bhgcPHui0KCIiIlPx7Kqvei5Eh9QKG2fPnkVubi4AYOrUqbh//75OiyIiIjIVXGfjqYYNG6JPnz5o0aIFhBCYO3cuHBwcitx20qRJWi2QiIjI0AghcPTqXWTmPHrpfV2/+xAAwwaWL1+OyZMnY8uWLZDJZIiPj4eFhepDZTIZwwYRERm9lCtZeP/b/Vrdp6W5iYcNX19fxMXFAQDMzMywY8cOVKpUSaeF6VpsbCxiY2NRWFio71KIiKicych+0qPhaGOBup5OL72/CnaWaOvv+dL7MVQaX4hNLpfroo5XLiYmBjExMcjJyYGzs7O+yyEionJEsTZG/arO+KV/Mz1XY/jKdNXXS5cuYcGCBTh79ixkMhnq1q2LYcOGoWbNmtquj4iIyOCYwqRObdJ4nY1t27bBz88Phw4dQv369REQEICDBw/C398fiYmJuqiRiIjIoDzNGqQmjXs2xo0bhxEjRmDWrFkq7Z9++inatGmjteKIiIgMkWIhLvZsqEfjno2zZ8+iX79+Ku19+/bFmTNntFIUERGRITOFy8Jrk8Zhw93dHampqSrtqamp5f4MFSIiInUIztnQiMbDKAMGDMBHH32Ev//+G6GhoZDJZNizZw9mz56NUaNG6aJGIiIig6KYs8GsoR6Nw8bEiRPh6OiIefPmYfz48QAALy8vTJkyBUOHDtV6gURERIZGLoUNpg11aBw2ZDIZRowYgREjRuDevXsAAEdHR60XRkREZKg4Z0MzZVpnQ4Ehg4iITBHnbGhG4wmiREREpk6xzAbDhnoYNoiIiDQklyZt6LeO8oJhg4iISENc1EszGoWN/Px8RERE4K+//tJVPURERAaPE0Q1o1HYsLS0xKlTp3iqDxEREdizoS6Nh1F69eqFpUuX6qIWIiKickHRs8GsoR6NT319/PgxfvjhByQmJiIoKAj29vZK98+fP19rxRERERmiZ/NDmTbUoXHYOHXqFBo3bgwAKnM3OLxCRESmgHM2NKNx2Ni1a5cu6iAiIio3BM9G0UiZT329ePEitm3bhocPHwJ4tpoaERGRsZNWEOUCEmrR+G26c+cOWrVqhddffx0dOnRARkYGAKB///686isREZkEufT3NXs21KFx2BgxYgQsLS1x9epV2NnZSe1du3ZFQkKCVosjIiIyRJyzoRmN52xs374d27ZtQ9WqVZXaa9eujStXrmitMCIiIkPFORua0bhnIzc3V6lHQ+H27duwtrbWSlFERESGTLBnQyMah42WLVti5cqV0m2ZTAa5XI45c+YgIiJCq8UREREZImmdDfZsqEXjYZQ5c+YgPDwcKSkpePz4McaOHYvTp0/j33//xd69e3VRIxERkUHhCqKa0Ths+Pn54cSJE1iyZAnMzc2Rm5uLd955BzExMahcubIuaiQiIhNTKBfov+IwzmTk6LuUIt17VACAczbUpXHYAABPT09MnTpV27W8UrGxsYiNjUVhYaG+SyEiohdcuZOLXedv6buMUtWu5KDvEsoFmSjDalxZWVlYunQpzp49C5lMhrp166JPnz5wdXXVRY06lZOTA2dnZ2RnZ8PJyUnf5RAREYCLmffQev5uOFpbYPVHzfRdTpHsrS1Qo6J96RsaMXU/QzXu2UhOTsbbb78NJycnBAUFAQC+/vprTJs2DZs2bUJYWFjZqyYiIsKzU0stzGUIqOKs32LopWkcNmJiYtClSxdpzgYAFBYWYvDgwYiJicGpU6e0XiQREZkWRZc7z/YwDhqf+nrp0iWMGjVKChoAYG5ujpEjR+LSpUtaLY6IiEyTkC7hTsZA47DRuHFjnD17VqX97NmzaNiwoTZqIiIiE/fs1FLGDWOg1jDKiRMnpP8PHToUw4YNw8WLF9Gs2ZNJOwcOHEBsbCxmzZqlmyqJiMikSD0bzBpGQa2zUczMzCCTyUq9jLxMJit3p5LybBQiIsNzOj0bb329B5UcrXFoQmt9l0PF0OrZKJcvX9ZaYURERKVhz4ZxUStseHt767oOIiIiCa+qalzKtILo9evXsXfvXmRmZkIulyvdN3ToUK0URkREpks8PfmVUcM4aBw2li1bhoEDB8LKygpubm5KM4VlMhnDBhERvTTBq6oaFY3DxqRJkzBp0iSMHz8eZmYanzlLRERUKo2vo0EGTeO08ODBA3Tr1o1Bg4iIdEaxzgY/aoyDxoexX79++N///qeLWoiIiAA8v4Ioh1GMgcbDKDNnzkTHjh2RkJCAevXqwdLSUun++fPna604IiIyVYoVRPVcBmmFxmFjxowZ2LZtG3x9fQFAZYIoERHRy+K1UYyLxmFj/vz5+PHHHxEdHa2DcoiIiAA519kwKhrP2bC2tkbz5s11UQsREREAPLs8BrOGUdA4bAwbNgyLFi3SRS1EREQAnp36yqxhHDQeRjl06BB27tyJLVu2wN/fX2WC6Lp167RWHBERmSYu6mVcNA4bFSpUwDvvvKOLWoiIiAA8G0YxY9YwCmVarpyIiEiXng2jMG0YA67NRkREBoeXmDcuGvds1KhRo8QxtL///vulCiIiIhK8OopR0ThsDB8+XOl2fn4+jh07hoSEBIwZM0ZbdRERkQnjOhvGReOwMWzYsCLbY2NjkZKS8tIFERERKSaIMmsYB63N2YiMjMTatWu1tTsiIjJh0gRRhg2joLWw8dtvv8HV1VVbuyMiIlPGq74aFY2HURo1aqQ0QVQIgRs3buDWrVtYvHixVosjIiLTJOc6G0ZF47DRuXNnpdtmZmZwd3dHeHg46tSpo626iIjIhAmOoxgVjcPG5MmTdVEHERGRhNdGMS5c1IuIiAwOz0YxLmr3bJiZmZV6QRyZTIaCgoKXLoqIiEwb19kwLmqHjfXr1xd73759+7Bo0SIpiRIREb2cpz0beq6CtEPtsPH222+rtJ07dw7jx4/H5s2b0aNHD0yfPl2rxRERkWnitVGMS5nmbKSnp2PAgAGoX78+CgoKkJqaihUrVqB69eraro+IiEyQnOtsGBWNwkZ2djY+/fRT1KpVC6dPn8aOHTuwefNmBAQE6Ko+IiIyQYoLsbFnwzioPYzy5ZdfYvbs2fD09MTq1auLHFYhIiLSBg6jGBeZUHNWp5mZGWxtbdG6dWuYm5sXu926deu0VtyrkJOTA2dnZ2RnZ8PJyUnf5RCRgfl+9yWsO3pd32WYnJyH+UjPfoSQ19yw+qNm+i6HiqHuZ6jaPRu9evUq9dTX8iQ2NhaxsbEoLCzUdylEZMC+3/03bt9/rO8yTFY1V1t9l0BaoHbPhrFizwYRlaThtO24+yAfX75XH1Uq8IPvVbIwk6FRdRdYWXD9SUOl9Z4NIiJTpPhzrHF1F9Sq5KDfYojKKcZFIiI1GNEoMtErx7BBRFQCEx9pJtIKhg0iohLw6qNEL49hg4ioJNJ6D4wbRGXFsEFEpAZGDaKyY9ggIioBZ2wQvTyGDSKiEigmiHIUhajsGDaIiNTAq48SlR3DBhFRCaSzUZg1iMqMYYOIqARcZoPo5TFsEBGVQHCKKNFLY9ggIlIDh1GIyo5hg4ioBBxGIXp5DBtERCV4NkGUXRtEZcWwQURUEsVy5fqtgqhcY9ggIlIDOzaIyo5hg4ioBDwbhejlMWwQEZVASMMo7NogKiuGDSKiEnAFUaKXx7BBRKQGZg2ismPYICIqgeBCG0QvjWGDiKgEUtRg1wZRmTFsEBGVgBNEiV4ewwYRERHpFMMGEZEaeDYKUdkxbBARFeP5yaHMGkRlx7BBRFSM509E4YXYiMqOYYOIiIh0imGDiKgYz6+wwX4NorJj2CAiKobSnA2mDaIyY9ggIiqGcs8G0wZRWTFsEBERkU4xbBARFUNw0gaRVjBsEBEVQ4BzNoi0gWGDiKgYvOArkXYwbBARqYEdG0Rlx7BBRKQGriBKVHYMG0RExVBarlx/ZRCVewwbREREpFMMG0RExeDZKETawbBBRFQM5WEUpg2ismLYICIqhtKaXswaRGXGsEFEREQ6xbBBRFQMwVW9iLSCYYOIqBgcRiHSDoYNIqJisGODSDsYNoiI1MCzUYjKjmGDiKg4z5/6yqxBVGYMG0RExVBa1EuPdRCVdwwbRETF4JwNIu1g2CAiUgOv+kpUdgwbRETFUDr1VW9VEJV/DBtERMV4flEvdmwQlR3DBhFRMThlg0g7GDaIiNTAORtEZcewQURUDJ6NQqQdDBtERMUQHEgh0gqGDSKi4jzNGhxBIXo5DBtERKVg1iB6OQwbRETFUAyicHIo0csxirCxZcsW+Pr6onbt2vjhhx/0XQ4RGQlOECXSDgt9F/CyCgoKMHLkSOzatQtOTk5o3Lgx3nnnHbi6uuq7NCIq5xQTRNmvQfRyyn3PxqFDh+Dv748qVarA0dERHTp0wLZt2/RdFhEZEY6iEL0cvYeN3bt3IyoqCl5eXpDJZNiwYYPKNosXL0aNGjVgY2ODwMBA/Pnnn9J96enpqFKlinS7atWquH79+qsonYiMnGIYRca+DaKXovdhlNzcXDRo0AB9+vTBu+++q3L/mjVrMHz4cCxevBjNmzfHd999h8jISJw5cwbVq1dXunaBgj4nc93MeYRjV7P09vxEpD3/5ubruwQio6D3sBEZGYnIyMhi758/fz769euH/v37AwAWLFiAbdu2YcmSJZg5cyaqVKmi1JNx7do1BAcHF7u/vLw85OXlSbdzcnK08CqeOXb1Lgb+fFSr+yQi/bIwZ88G0cvQe9goyePHj3HkyBGMGzdOqb1t27bYt28fAKBp06Y4deoUrl+/DicnJ2zduhWTJk0qdp8zZ87E1KlTdVZzBTtLBHm76Gz/RPTqtQ/w1HcJROWaQYeN27dvo7CwEB4eHkrtHh4euHHjBgDAwsIC8+bNQ0REBORyOcaOHQs3N7di9zl+/HiMHDlSup2Tk4Nq1appreZmr7nht0GhWtsfERFReWfQYUPhxTkYQgiltk6dOqFTp05q7cva2hrW1tZarY+IiIiKp/ezUUpSsWJFmJubS70YCpmZmSq9HURERGSYDDpsWFlZITAwEImJiUrtiYmJCA3lUAUREVF5oPdhlPv37+PixYvS7cuXLyM1NRWurq6oXr06Ro4ciZ49eyIoKAghISH4/vvvcfXqVQwcOFCPVRMREZG69B42UlJSEBERId1WTN7s3bs3li9fjq5du+LOnTuYNm0aMjIyEBAQgK1bt8Lb21tfJRMREZEGZKKoVbFMSE5ODpydnZGdnQ0nJyd9l0NERFRuqPsZatBzNoiIiKj8Y9ggIiIinTLZsBEbGws/Pz80adJE36UQEREZNc7Z4JwNIiKiMuGcDSIiIjIIej/1Vd8UHTvavvorERGRsVN8dpY2SGLyYePevXsAoNWLsREREZmSe/fuwdnZudj7TX7OhlwuR3p6OhwdHVUu+FZWiivJpqWlcR6IHvE4GAYeB8PBY2EYjOk4CCFw7949eHl5wcys+JkZJt+zYWZmhqpVq+pk305OTuX+G8kY8DgYBh4Hw8FjYRiM5TiU1KOhwAmiREREpFMMG0RERKRTDBs6YG1tjcmTJ8Pa2lrfpZg0HgfDwONgOHgsDIMpHgeTnyBKREREusWeDSIiItIphg0iIiLSKYYNIiIi0imGDSIiItIphg0tW7x4MWrUqAEbGxsEBgbizz//1HdJ5dru3bsRFRUFLy8vyGQybNiwQel+IQSmTJkCLy8v2NraIjw8HKdPn1baJi8vD5988gkqVqwIe3t7dOrUCdeuXVPaJisrCz179oSzszOcnZ3Rs2dP3L17V8evrvyYOXMmmjRpAkdHR1SqVAmdO3fG+fPnlbbhsdC9JUuWoH79+tJiUCEhIYiPj5fu5zHQj5kzZ0Imk2H48OFSG4/FCwRpTVxcnLC0tBT/93//J86cOSOGDRsm7O3txZUrV/RdWrm1detWMWHCBLF27VoBQKxfv17p/lmzZglHR0exdu1acfLkSdG1a1dRuXJlkZOTI20zcOBAUaVKFZGYmCiOHj0qIiIiRIMGDURBQYG0Tfv27UVAQIDYt2+f2LdvnwgICBAdO3Z8VS/T4LVr104sW7ZMnDp1SqSmpoq33npLVK9eXdy/f1/ahsdC9zZt2iR+//13cf78eXH+/Hnx2WefCUtLS3Hq1CkhBI+BPhw6dEj4+PiI+vXri2HDhkntPBbKGDa0qGnTpmLgwIFKbXXq1BHjxo3TU0XG5cWwIZfLhaenp5g1a5bU9ujRI+Hs7Cy+/fZbIYQQd+/eFZaWliIuLk7a5vr168LMzEwkJCQIIYQ4c+aMACAOHDggbbN//34BQJw7d07Hr6p8yszMFABEcnKyEILHQp9cXFzEDz/8wGOgB/fu3RO1a9cWiYmJIiwsTAobPBaqOIyiJY8fP8aRI0fQtm1bpfa2bdti3759eqrKuF2+fBk3btxQes+tra0RFhYmvedHjhxBfn6+0jZeXl4ICAiQttm/fz+cnZ0RHBwsbdOsWTM4Ozvz2BUjOzsbAODq6gqAx0IfCgsLERcXh9zcXISEhPAY6EFMTAzeeusttG7dWqmdx0KVyV+ITVtu376NwsJCeHh4KLV7eHjgxo0beqrKuCne16Le8ytXrkjbWFlZwcXFRWUbxeNv3LiBSpUqqey/UqVKPHZFEEJg5MiRaNGiBQICAgDwWLxKJ0+eREhICB49egQHBwesX78efn5+0ocPj8GrERcXh6NHj+Lw4cMq9/HnQRXDhpa9eJl6IYTWLl1PRSvLe/7iNkVtz2NXtCFDhuDEiRPYs2ePyn08Frrn6+uL1NRU3L17F2vXrkXv3r2RnJws3c9joHtpaWkYNmwYtm/fDhsbm2K347F4hsMoWlKxYkWYm5urpM3MzEyVdEva4enpCQAlvueenp54/PgxsrKyStzm5s2bKvu/desWj90LPvnkE2zatAm7du1C1apVpXYei1fHysoKtWrVQlBQEGbOnIkGDRpg4cKFPAav0JEjR5CZmYnAwEBYWFjAwsICycnJ+Prrr2FhYSG9TzwWzzBsaImVlRUCAwORmJio1J6YmIjQ0FA9VWXcatSoAU9PT6X3/PHjx0hOTpbe88DAQFhaWiptk5GRgVOnTknbhISEIDs7G4cOHZK2OXjwILKzs3nsnhJCYMiQIVi3bh127tyJGjVqKN3PY6E/Qgjk5eXxGLxCrVq1wsmTJ5Gamip9BQUFoUePHkhNTcVrr73GY/GiVz8n1XgpTn1dunSpOHPmjBg+fLiwt7cX//zzj75LK7fu3bsnjh07Jo4dOyYAiPnz54tjx45JpxPPmjVLODs7i3Xr1omTJ0+KDz74oMjTy6pWrSr++OMPcfToUfHmm28WeXpZ/fr1xf79+8X+/ftFvXr1yuXpZboyaNAg4ezsLJKSkkRGRob09eDBA2kbHgvdGz9+vNi9e7e4fPmyOHHihPjss8+EmZmZ2L59uxCCx0Cfnj8bRQgeixcxbGhZbGys8Pb2FlZWVqJx48bSqYFUNrt27RIAVL569+4thHhyitnkyZOFp6ensLa2Fi1bthQnT55U2sfDhw/FkCFDhKurq7C1tRUdO3YUV69eVdrmzp07okePHsLR0VE4OjqKHj16iKysrFf0Kg1fUccAgFi2bJm0DY+F7vXt21f6/eLu7i5atWolBQ0heAz06cWwwWOhjJeYJyIiIp3inA0iIiLSKYYNIiIi0imGDSIiItIphg0iIiLSKYYNIiIi0imGDSIiItIphg0iIiLSKYYNIiIi0imGDSIq0j///AOZTIbU1FR9lyI5d+4cmjVrBhsbGzRs2PCVP394eDiGDx/+yp+XqLxj2CAyUNHR0ZDJZJg1a5ZS+4YNG8rd5aW1ZfLkybC3t8f58+exY8eOV/7869atw/Tp07W2P4YXMhUMG0QGzMbGBrNnz1a5DHV59vjx4zI/9tKlS2jRogW8vb3h5ub2ymtydXWFo6Oj1p6XyFQwbBAZsNatW8PT0xMzZ84sdpspU6aoDCksWLAAPj4+0u3o6Gh07twZM2bMgIeHBypUqICpU6eioKAAY8aMgaurK6pWrYoff/xRZf/nzp1DaGgobGxs4O/vj6SkJKX7z5w5gw4dOsDBwQEeHh7o2bMnbt++Ld0fHh6OIUOGYOTIkahYsSLatGlT5OuQy+WYNm0aqlatCmtrazRs2BAJCQnS/TKZDEeOHMG0adMgk8kwZcqUIvejeL4hQ4agQoUKcHNzw+eff47nLwPl4+ODL774AtHR0XB2dsaAAQMAAGvXroW/vz+sra3h4+ODefPmqez7+Z6Ix48fY+zYsahSpQrs7e0RHBys8v7s3bsXYWFhsLOzg4uLC9q1a4esrCxER0cjOTkZCxcuhEwmg0wmwz///IOsrCz06NED7u7usLW1Re3atbFs2bIiXytRecGwQWTAzM3NMWPGDCxatAjXrl17qX3t3LkT6enp2L17N+bPn48pU6agY8eOcHFxwcGDBzFw4EAMHDgQaWlpSo8bM2YMRo0ahWPHjiE0NBSdOnXCnTt3AAAZGRkICwtDw4YNkZKSgoSEBNy8eRNdunRR2seKFStgYWGBvXv34rvvviuyvoULF2LevHmYO3cuTpw4gXbt2qFTp064cOGC9Fz+/v4YNWoUMjIyMHr06GJfq+L5Dh48iK+//hpfffUVfvjhB6Vt5syZg4CAABw5cgQTJ07EkSNH0KVLF3Tr1g0nT57ElClTMHHiRCxfvrzY5+nTpw/27t2LuLg4nDhxAu+//z7at28v1ZyamopWrVrB398f+/fvx549exAVFYXCwkIsXLgQISEhGDBgADIyMpCRkYFq1aph4sSJOHPmDOLj43H27FksWbIEFStWLLYGonJBz1edJaJi9O7dW7z99ttCCCGaNWsm+vbtK4QQYv369eL5H93JkyeLBg0aKD32q6++Et7e3kr78vb2FoWFhVKbr6+veOONN6TbBQUFwt7eXqxevVoIIcTly5cFADFr1ixpm/z8fFG1alUxe/ZsIYQQEydOFG3btlV67rS0NAFAnD9/Xgjx5NLbDRs2LPX1enl5if/+979KbU2aNBGDBw+Wbjdo0EBMnjy5xP2EhYWJunXrCrlcLrV9+umnom7dutJtb29v0blzZ6XHde/eXbRp00apbcyYMcLPz09p34rLiF+8eFHIZDJx/fp1pce0atVKjB8/XgghxAcffCCaN29eYq3PX5ZcCCGioqJEnz59SnyNROUNezaIyoHZs2djxYoVOHPmTJn34e/vDzOzZz/yHh4eqFevnnTb3Nwcbm5uyMzMVHpcSEiI9H8LCwsEBQXh7NmzAIAjR45g165dcHBwkL7q1KkD4Mn8CoWgoKASa8vJyUF6ejqaN2+u1N68eXPpuTTRrFkzpUm0ISEhuHDhAgoLC4ut6ezZs0U+/4uPUzh69CiEEHj99deVXn9ycrL02hU9G5oYNGgQ4uLi0LBhQ4wdOxb79u3T6PFEhshC3wUQUelatmyJdu3a4bPPPkN0dLTSfWZmZkrzEQAgPz9fZR+WlpZKt2UyWZFtcrm81HoUH+RyuRxRUVGYPXu2yjaVK1eW/m9vb1/qPp/fr4IQQmdn3rxYU1HP9eL7+jy5XA5zc3McOXIE5ubmSvc5ODgAAGxtbTWuKzIyEleuXMHvv/+OP/74A61atUJMTAzmzp2r8b6IDAV7NojKiVmzZmHz5s0qf+m6u7vjxo0bSh+M2lwb48CBA9L/CwoKcOTIEan3onHjxjh9+jR8fHxQq1YtpS91AwYAODk5wcvLC3v27FFq37dvH+rWrftSNStu165dWyUUPM/Pz6/I53/99deLfFyjRo1QWFiIzMxMldfu6ekJAKhfv36Jp+haWVkV2Wvi7u6O6Oho/Pzzz1iwYAG+//77El8vkaFj2CAqJ+rVq4cePXpg0aJFSu3h4eG4desWvvzyS1y6dAmxsbGIj4/X2vPGxsZi/fr1OHfuHGJiYpCVlYW+ffsCAGJiYvDvv//igw8+wKFDh/D3339j+/bt6Nu3b5EfoiUZM2YMZs+ejTVr1uD8+fMYN24cUlNTMWzYMI1rTktLw8iRI3H+/HmsXr0aixYtKnU/o0aNwo4dOzB9+nT89ddfWLFiBb755ptiJ6K+/vrr6NGjB3r16oV169bh8uXLOHz4MGbPno2tW7cCAMaPH4/Dhw9j8ODBOHHiBM6dO4clS5ZIZ+v4+Pjg4MGD+Oeff3D79m3I5XJMmjQJGzduxMWLF3H69Gls2bKlTIGLyJAwbBCVI9OnT1fp2q9bty4WL16M2NhYNGjQAIcOHSrxTA1NzZo1C7Nnz0aDBg3w559/YuPGjdLZEV5eXti7dy8KCwvRrl07BAQEYNiwYXB2dlaaH6KOoUOHYtSoURg1ahTq1auHhIQEbNq0CbVr19a45l69euHhw4do2rQpYmJi8Mknn+Cjjz4q8TGNGzfGr7/+iri4OAQEBGDSpEmYNm2ayrDV85YtW4ZevXph1KhR8PX1RadOnXDw4EFUq1YNwJNAsn37dhw/fhxNmzZFSEgINm7cCAuLJyPYo0ePhrm5Ofz8/ODu7o6rV6/CysoK48ePR/369dGyZUuYm5sjLi5O4/eAyJDIREmDkkRE5Ux4eDgaNmyIBQsWaH3fISEhaNWqFb744gut75vImLFng4ioFHl5eUhJScHp06fh7++v73KIyh2GDSKiUsTHx+PNN99EVFQU3nvvPX2XQ1TucBiFiIiIdIo9G0RERKRTDBtERESkUwwbREREpFMMG0RERKRTDBtERESkUwwbREREpFMMG0RERKRTDBtERESkU/8PTPYjKtb7Dj0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 600x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "AI_cordis = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/cordis_Kwds_AI.parquet\")\n",
    "counts = sorted([el[0] for el in AI_cordis.select(\"Kwd_count\").collect()])\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "plt.figure(figsize=(6,4))\n",
    "plt.semilogy(np.linspace(0,AI_cordis.count(), len(counts)), counts)\n",
    "plt.xlabel(\"Number of projects\")\n",
    "plt.ylabel(\"Number of keywords\")\n",
    "plt.title(\"Cumulative number of projects (CORDIS)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84f98bc4-4764-4d45-a689-ffac7cc3150d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of projects in cordis with 1+ Kwds 4301\n",
      "Number of projects in cordis with 3+ Kwds 852\n"
     ]
    }
   ],
   "source": [
    "AI_cordis = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/cordis_Kwds_AI.parquet\")\n",
    "print(\"Number of projects in cordis with 1+ Kwds\", AI_cordis.count())\n",
    "AI_cordis = spark.read.parquet(\"file:///export/usuarios_ml4ds/jarenas/github/IntelComp/KPIs/WP3-pipelines/datasets/AI/cordis_Kwds3_AI.parquet\")\n",
    "print(\"Number of projects in cordis with 3+ Kwds\", AI_cordis.count())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}