vantage6Client_DCR_Pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ypG7Bzar2t3K",
   "metadata": {
    "id": "ypG7Bzar2t3K"
   },
   "source": [
    "## **Install requirements**"
   ]
  },
  {
   "cell_type": "code",
   "id": "xM4xBiWQ2saU",
   "metadata": {
    "id": "xM4xBiWQ2saU"
   },
   "source": [
    "!pip install vantage6==3.7.3"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "CZdJ6cxa25QU",
   "metadata": {
    "id": "CZdJ6cxa25QU"
   },
   "source": [
    "**Imports**"
   ]
  },
  {
   "cell_type": "code",
   "id": "vf4XlLyO28Ej",
   "metadata": {
    "id": "vf4XlLyO28Ej"
   },
   "source": [
    "from vantage6.client import Client\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "Ng7ay-Jz3PtZ",
   "metadata": {
    "id": "Ng7ay-Jz3PtZ"
   },
   "source": [
    "## **Vantage6 Server Connection**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "xEn1hy4E4Iz4",
   "metadata": {
    "id": "xEn1hy4E4Iz4"
   },
   "source": [
    "Fill here the Vantage6 server information and credentials"
   ]
  },
  {
   "cell_type": "code",
   "id": "ac7cc209",
   "metadata": {
    "id": "ac7cc209"
   },
   "source": [
    "# config.py\n",
    "\n",
    "server_url = \"http://35.157.139.38\"\n",
    "server_port = 443 # This is specified when you first created the server\n",
    "server_api = \"/api\" # This is specified when you first created the server\n",
    "\n",
    "username = \"Varsha\"\n",
    "password = \"\"\n",
    "\n",
    "organization_key = None"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "BzF1u0YL4xLG",
   "metadata": {
    "id": "BzF1u0YL4xLG"
   },
   "source": [
    "Authenticate with the server"
   ]
  },
  {
   "cell_type": "code",
   "id": "ce45b85c",
   "metadata": {
    "id": "ce45b85c"
   },
   "source": [
    "# Initialize the client object, and run the authentication\n",
    "client = Client(server_url, server_port, server_api, verbose=True)\n",
    "client.authenticate(username, password)\n",
    "\n",
    "# Optional: setup the encryption, if you have an organization_key\n",
    "client.setup_encryption(organization_key)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "OgaDFsIP42oO",
   "metadata": {
    "id": "OgaDFsIP42oO"
   },
   "source": [
    "Check the node status"
   ]
  },
  {
   "cell_type": "code",
   "id": "59586c6a",
   "metadata": {
    "id": "59586c6a"
   },
   "source": [
    "org = client.node.list()\n",
    "org"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "a9e4988c",
   "metadata": {},
   "source": [
    "# delete all tasks \n",
    "# Clear the list of tasks (any tasks that cannot be completed will constantly run otherwise)\n",
    "tasks = client.task.list(per_page = 1000)\n",
    "#print(tasks)\n",
    "\n",
    "t = tasks['data']\n",
    "for task in t:\n",
    "    #print(task)\n",
    "    client.task.delete(id_=task['id'])"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "0905c5df",
   "metadata": {},
   "source": [
    "result = client.result.get(74)\n",
    "print(result)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "UrLruADn5Hmd",
   "metadata": {
    "id": "UrLruADn5Hmd"
   },
   "source": [
    "## **Step 1 - Query required data**\n"
   ]
  },
  {
   "cell_type": "code",
   "id": "9521d9c1",
   "metadata": {
    "id": "9521d9c1"
   },
   "source": [
    "# Submit new task and wait for results\n",
    "\n",
    "time_col_name = 'metastasisdays'\n",
    "event_col_name = 'metastasis'\n",
    "\n",
    "input_ = {\n",
    "    'master': True,\n",
    "    'method': 'master',\n",
    "    'kwargs': {'feature_type': 'Radiomics',  ## Clinical or Radiomics or Combined\n",
    "               'time_col': time_col_name,\n",
    "               'outcome_col': event_col_name,\n",
    "               'expl_vars': ['Fszm_sze', 'Fdzm_lgze', 'Fstat_skew', 'Fcm_info_corr_1'], ## required for radiomics and combined model, replace . with _\n",
    "               'organization_ids': [2, 3, 4] # here list the organizations ids\n",
    "               }\n",
    "}\n",
    "\n",
    "task = client.task.create(name=\"Querying task\",\n",
    "                               description=\"Send SPARQL queries to fetch required data based on the given feature type\",\n",
    "                               image=\"varshagouthamchand/dcr_sparql_query\",\n",
    "                               collaboration=1,\n",
    "                               input=input_,\n",
    "                               organizations=[2], ## aggregator node\n",
    "                               database='rdf')\n",
    "\n",
    "#print(\"Waiting for results\")\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    #print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "result = result_info['data'][0]['result']\n",
    "print(result)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "fa019fcd",
   "metadata": {},
   "source": [
    "result = client.result.get(423)\n",
    "print(result)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "vtyfFa5m6dG7",
   "metadata": {
    "id": "vtyfFa5m6dG7"
   },
   "source": [
    "## **Step 2 - Distributed Feature Selection**\n",
    "\n",
    "Apply on the distributed training set **Correlation-based Feature Selection** (Step  2.1) and **Cox Proportional Hazards Regression Model with LASSO regularization** (Step 2.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "yXmZbGSP7fCo",
   "metadata": {
    "id": "yXmZbGSP7fCo"
   },
   "source": [
    "### **Step 2.1 - Correlation-based Feature Selection**\n"
   ]
  },
  {
   "cell_type": "code",
   "id": "6a63ca2c",
   "metadata": {
    "id": "6a63ca2c",
    "scrolled": false
   },
   "source": [
    "time_col_name = 'recurrencedays'\n",
    "event_col_name = 'recurrence'\n",
    "\n",
    "# Submit new task and wait for results\n",
    "input_ = {\n",
    "    'master': True,\n",
    "    'method': 'master',\n",
    "    'kwargs': {'expl_vars': [] ## leave it empty as it fetches all the common features or add features to be considered for correlation\n",
    "             , 'outcome_col': event_col_name\n",
    "             , 'organization_ids': [2, 7, 8] \n",
    "             , 'roitype': \"Primary\"  ## Primary or Node or Combined\n",
    "             , 'feature_type': 'Radiomics'  ## Radiomics or Combined\n",
    "             , 'oropharynx': 'yes'\n",
    "                }\n",
    "    }\n",
    "\n",
    "task = client.task.create(name=\"Correlation-base Feature Selection\",\n",
    "                               description=\"testing corr\",\n",
    "                               image=\"varshagouthamchand/dcr_sparql_corr\",\n",
    "                               collaboration=3,\n",
    "                               input=input_,\n",
    "                               organizations=[2],\n",
    "                               database='rdf')\n",
    "\n",
    "print(\"Waiting for results\")\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "corr_result = result_info['data'][0]['result']\n",
    "print(corr_result)\n",
    "corr_result['correlationMatrix'].to_csv('corr_result_csv.csv')\n",
    "best_subset = corr_result['best_subset']\n",
    "print(best_subset)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "id": "DVvwelSn82T3",
   "metadata": {
    "id": "DVvwelSn82T3"
   },
   "source": [
    "### **Step 2.2.1 - Cox Regression with LASSO**\n",
    "\n",
    "Train a Cox Regression Proportional Hazard Model with LASSO Regularization on the training set of each node using as predictor the best subset in output from the CFS (Step 2.1).\n"
   ]
  },
  {
   "cell_type": "code",
   "id": "f32d897f",
   "metadata": {
    "id": "f32d897f"
   },
   "source": [
    "time_col_name = 'metastasisdays'\n",
    "event_col_name = 'metastasis'\n",
    "\n",
    "# Submit new task and wait for results\n",
    "input_ = {\n",
    "    'master': 1,\n",
    "    'method': 'master',\n",
    "    'kwargs': {\n",
    "          'expl_vars': ['Frlm_rlnu_norm', 'Fmorph_sph_sphericity', 'Fszm_zlnu_norm', 'Fcm_corr']\n",
    "        , 'time_col': time_col_name\n",
    "        , 'outcome_col': event_col_name\n",
    "        , 'feature_type': 'Radiomics' ## Clinical or Radiomics or Combined\n",
    "        , 'oropharynx': 'yes' ##yes or no\n",
    "        , 'roitype': \"Node\" ## Primary or Node\n",
    "        , 'n_lambda': 50\n",
    "        , 'lambda_range': (0, 50)\n",
    "        , 'beta_start': None\n",
    "        , 'epsilon': 0.00000001\n",
    "        , 'epochs': 150\n",
    "        , 'organization_ids': [2, 7, 8]}\n",
    "}\n",
    "\n",
    "task = client.task.create(name=\"Lasso Cox model\",\n",
    "                               description=\"Train a Cox Regression Proportional Hazard Model with LASSO Regularization on the training set of each node using as predictor the best subset in output from the CFS\",\n",
    "                               image=\"varshagouthamchand/dcr_sparql_lasso\",\n",
    "                               collaboration=3,\n",
    "                               input=input_,\n",
    "                               organizations=[2],\n",
    "                               database='rdf')\n",
    "\n",
    "\n",
    "print(\"Waiting for results\")\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "lasso_results = result_info['data'][0]['result']\n",
    "lasso_path = lasso_results['model']\n",
    "print(lasso_path)\n",
    "lasso_path.to_csv('lasso_path.csv')\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "617a3c93",
   "metadata": {},
   "source": [
    "result = client.result.get(216)\n",
    "print(result)\n",
    "lasso_results = result['result']\n",
    "print(lasso_results)\n",
    "lasso_path = lasso_results['model']\n",
    "print(lasso_path)\n",
    "lasso_path.to_csv('lasso_path.csv')"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "27faee44",
   "metadata": {},
   "source": "## **Coxph** (if needed to run individually)",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "e2b647b0",
   "metadata": {
    "scrolled": false
   },
   "source": [
    "time_col_name = 'overallsurvivaldays'\n",
    "event_col_name = 'survival'\n",
    "expl_subset = ['lp_Clinical_all', 'lp_Radiomics_Primary', 'lp_Radiomics_Node']\n",
    "#expl_subset = ['treatment_Chemotherapy', 'N1orLower', 'hpv_HPV Negative', 'hpv_HPV Positive']\n",
    "#expl_subset = ['Fszm_lgze', 'Fcm_corr']\n",
    "\n",
    "# Submit new task and wait for results\n",
    "input_ = {\n",
    "    'master': 1,\n",
    "    'method': 'master',\n",
    "    'kwargs': {\n",
    "         # 'expl_vars':  best_subset or combined_strings\n",
    "          'expl_vars': expl_subset\n",
    "        , 'time_col': time_col_name\n",
    "        , 'outcome_col': event_col_name\n",
    "        , 'feature_type': 'LP'\n",
    "        , 'oropharynx': 'yes'\n",
    "        , 'roitype': \"Primary\"\n",
    "        , 'organization_ids': [2, 7, 8]}\n",
    "}\n",
    "\n",
    "task = client.task.create(name=\"Coxph model\",\n",
    "                               description=\"Train a Cox Regression Proportional Hazard Model using as predictor the subsets in output from LASSO\",\n",
    "                               image=\"varshagouthamchand/dcr_sparql_coxph\",\n",
    "                               collaboration=1,\n",
    "                               input=input_,\n",
    "                               organizations=[2],\n",
    "                               database='rdf')\n",
    "\n",
    "\n",
    "print(\"Waiting for results\")\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "cox_results = result_info['data'][0]['result']\n",
    "print(cox_results)\n",
    "#full_cox_results = cox_results['results']\n",
    "coeff = cox_results['Coef'].to_dict()\n",
    "print(coeff)\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "eb37db75",
   "metadata": {},
   "source": [
    "## **Leave one out cross validation for optimal feature selection**"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "387e0708",
   "metadata": {},
   "source": [
    "# coxph task \n",
    "\n",
    "def coxph(organization_ids, column_names):\n",
    "    \"\"\"\"\"\"\n",
    "    time_col_name = 'overallsurvivaldays'\n",
    "    event_col_name = 'survival'\n",
    "    \n",
    "    input_ = {\n",
    "        'master': 1,\n",
    "        'method': 'master',\n",
    "        'kwargs': {\n",
    "             # 'expl_vars':  best_subset or combined_strings\n",
    "              'expl_vars': column_names\n",
    "            , 'time_col': time_col_name\n",
    "            , 'outcome_col': event_col_name\n",
    "            , 'feature_type': 'Radiomics'\n",
    "            , 'oropharynx': 'yes'\n",
    "            , 'roitype': \"Primary\"\n",
    "            , 'organization_ids': organization_ids}\n",
    "    }\n",
    "\n",
    "    task = client.task.create(name=\"Coxph model\",\n",
    "                                   description=\"Train a Cox Regression Proportional Hazard Model using as predictor the subsets in output from LASSO\",\n",
    "                                   image=\"varshagouthamchand/dcr_sparql_coxph\",\n",
    "                                   collaboration=1,\n",
    "                                   input=input_,\n",
    "                                   organizations=[2],\n",
    "                                   database='rdf')\n",
    "\n",
    "\n",
    "    print(\"Waiting for results\")\n",
    "    task_id = task['id']\n",
    "    task_info = client.task.get(task_id)\n",
    "    while not task_info.get(\"complete\"):\n",
    "        task_info = client.task.get(task_id, include_results=True)\n",
    "        print(\"Waiting for results\")\n",
    "        time.sleep(3)\n",
    "\n",
    "    print(\"Results are ready!\")\n",
    "\n",
    "    result_id = task_info['id']\n",
    "    result_info = client.result.list(task=result_id)\n",
    "\n",
    "    cox_results = result_info['data'][0]['result']\n",
    "    #print(cox_results)\n",
    "    coeff = cox_results['Coef'].to_dict()\n",
    "    return coeff, cox_results\n",
    "    "
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "c1873437",
   "metadata": {},
   "source": [
    "# validate task\n",
    "\n",
    "def validation(output_data, organization_ids):\n",
    "    \"\"\"\"\"\"\n",
    "    time_col_name = 'overallsurvivaldays'\n",
    "    event_col_name = 'survival'\n",
    "\n",
    "    input_ = {\n",
    "            \"master\": True,\n",
    "            \"method\": \"master\",\n",
    "            # kwargs which are inserted into the algorithm\n",
    "            'kwargs': {\n",
    "                  'coefficients': output_data\n",
    "                , 'time_col': time_col_name\n",
    "                , 'outcome_col': event_col_name\n",
    "                , 'feature_type': 'Radiomics'\n",
    "                , 'oropharynx': 'yes'\n",
    "                , 'roitype': \"Primary\"\n",
    "                , 'organization_ids': organization_ids\n",
    "            }\n",
    "        }\n",
    "\n",
    "    # Send the task to the central server\n",
    "    task = client.task.create(name='validation',\n",
    "                                   description=\"test validation\",\n",
    "                                   collaboration=1,\n",
    "                                   organizations=[2],\n",
    "                                   image=\"varshagouthamchand/dcr_sparql_validation\",\n",
    "                                   input=input_,\n",
    "                                   database='rdf'\n",
    "                                   )\n",
    "    task_id = task['id']\n",
    "    task_info = client.task.get(task_id)\n",
    "    while not task_info.get(\"complete\"):\n",
    "        task_info = client.task.get(task_id, include_results=True)\n",
    "        print(\"Waiting for results\")\n",
    "        time.sleep(3)\n",
    "\n",
    "    print(\"Results are ready!\")\n",
    "\n",
    "    result_id = task_info['id']\n",
    "    result_info = client.result.list(task=result_id)\n",
    "\n",
    "    validation_result = result_info['data'][0]['result']\n",
    "    output = validation_result['cindex'][0]\n",
    "    return output"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "c92f684a",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "#cox regression with selected lamba values and leave one out validation\n",
    "#result for each run to be saved in a text file\n",
    "with open('LOOCV.txt', 'a') as f:\n",
    "    f.write('\\nCross Validation results\\n')\n",
    "organization_list = [2, 7, 8]\n",
    "\n",
    "c_index = []\n",
    "global_cindex = 0\n",
    "# perform cox regression\n",
    "for val_id in organization_list:\n",
    "    print(val_id)\n",
    "    with open('LOOCV.txt', 'a') as f:\n",
    "        f.write(f'Val_id:{val_id}\\n')\n",
    "    train_ids = organization_list.copy()\n",
    "    train_ids.remove(val_id)\n",
    "    print(train_ids)\n",
    "    with open('LOOCV.txt', 'a') as f:\n",
    "        f.write(f'train_ids:{train_ids}\\n')\n",
    "    coeff, cox_results = coxph(organization_ids=train_ids, column_names=['Fmorph_pca_elongation', 'Fcm_inv_var'])\n",
    "    print(coeff, cox_results)\n",
    "    with open('LOOCV.txt', 'a') as f:\n",
    "        f.write(f'coeff:{coeff}\\n')\n",
    "        f.write(f'cox_results:{cox_results}\\n')\n",
    "\n",
    "    # run validation\n",
    "    output = validation(output_data=[coeff], organization_ids=[val_id])\n",
    "    c_index.append(output)\n",
    "    print(c_index)\n",
    "    with open('LOOCV.txt', 'a') as f:\n",
    "        f.write(f'c_index:{c_index}\\n\\n')\n",
    "\n",
    "global_cindex = np.mean(c_index)\n",
    "print(global_cindex)\n",
    "with open('LOOCV.txt', 'a') as f:\n",
    "    f.write(f'global_cindex:{global_cindex}\\n')"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "3a8e1ec3",
   "metadata": {},
   "source": [],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "377c3ff1",
   "metadata": {},
   "source": "## **Step 2.2.2 - validation** (if needed to run individually)",
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "252ad2c1",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "time_col_name = 'recurrencedays'\n",
    "event_col_name = 'recurrence'\n",
    "output_data = {'lp_Clinical_all': 0.58609, 'lp_Radiomics_Primary': 0.11941, 'lp_Radiomics_Node': 1.02146}\n",
    " \n",
    "\n",
    "input_ = {\n",
    "        \"master\": True,\n",
    "        \"method\": \"master\",\n",
    "        # kwargs which are inserted into the algorithm\n",
    "        'kwargs': {\n",
    "            'coefficients': [output_data]\n",
    "            , 'time_col': time_col_name\n",
    "            , 'outcome_col': event_col_name\n",
    "            , 'feature_type': 'LP'\n",
    "            , 'oropharynx': 'yes'\n",
    "            , 'roitype': \"Primary\"\n",
    "            , 'organization_ids': [7]\n",
    "        }\n",
    "    }\n",
    "\n",
    "# Send the task to the central server\n",
    "task = client.task.create(name='validation',\n",
    "                               description=\"test validation\",\n",
    "                               collaboration=3,\n",
    "                               organizations=[7],\n",
    "                               image=\"varshagouthamchand/dcr_sparql_validation\",\n",
    "                               input=input_,\n",
    "                               database='rdf'\n",
    "                               )\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "validation_result = result_info['data'][0]['result']\n",
    "output = validation_result['cindex'][0]\n",
    "print(output)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "47c36cb6",
   "metadata": {},
   "source": [
    "# External validate task to run on multiple organizations and get the global c-index\n",
    "\n",
    "def validation(output_data, organization_ids):\n",
    "    \"\"\"\"\"\"\n",
    "    time_col_name = 'metastasisdays'\n",
    "    event_col_name = 'metastasis'\n",
    "    \n",
    "    input_ = {\n",
    "            \"master\": True,\n",
    "            \"method\": \"master\",\n",
    "            # kwargs which are inserted into the algorithm\n",
    "            'kwargs': {\n",
    "                  'coefficients': output_data\n",
    "                , 'time_col': time_col_name\n",
    "                , 'outcome_col': event_col_name\n",
    "                , 'feature_type': 'LP'\n",
    "                , 'oropharynx': 'yes'\n",
    "                , 'roitype': \"Primary\"\n",
    "                , 'organization_ids': organization_ids\n",
    "            }\n",
    "        }\n",
    "\n",
    "    # Send the task to the central server\n",
    "    task = client.task.create(name='validation',\n",
    "                                   description=\"test validation\",\n",
    "                                   collaboration=1,\n",
    "                                   organizations=[5],\n",
    "                                   image=\"varshagouthamchand/dcr_sparql_validation\",\n",
    "                                   input=input_,\n",
    "                                   database='rdf'\n",
    "                                   )\n",
    "    task_id = task['id']\n",
    "    task_info = client.task.get(task_id)\n",
    "    while not task_info.get(\"complete\"):\n",
    "        task_info = client.task.get(task_id, include_results=True)\n",
    "        print(\"Waiting for results\")\n",
    "        time.sleep(3)\n",
    "\n",
    "    print(\"Results are ready!\")\n",
    "\n",
    "    result_id = task_info['id']\n",
    "    result_info = client.result.list(task=result_id)\n",
    "\n",
    "    validation_result = result_info['data'][0]['result']\n",
    "    output = validation_result['cindex'][0]\n",
    "    return output, validation_result"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "b7244c72",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "organization_list = [3, 4, 5]\n",
    "coeff = {'lp_Clinical_all': 0.28912, 'lp_Radiomics_Primary': 0.26349, 'lp_Radiomics_Node': 0.4964}\n",
    "\n",
    "c_index = []\n",
    "global_cindex = 0\n",
    "# perform cox regression\n",
    "for val_id in organization_list:\n",
    "    # run validation\n",
    "    output, validation_result = validation(output_data=[coeff], organization_ids=[val_id])\n",
    "    c_index.append(output)\n",
    "    print(val_id, validation_result, output)\n",
    "\n",
    "global_cindex = np.mean(c_index)\n",
    "print(c_index)\n",
    "print(global_cindex)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "7abaea0f",
   "metadata": {},
   "source": [
    "# Submit task to get linear predictors in individual organizations \n",
    "# the coefficients from all three models to be combined as a dictionary and passed as input\n",
    "\n",
    "time_col_name = 'overallsurvivaldays'\n",
    "event_col_name = 'survival'\n",
    "coefficients = {'Clinical_all': {'treatment_Chemotherapy': -0.13098, 'N1orLower': -0.94049, 'hpv_HPV Negative': -0.4653, 'hpv_HPV Positive': -2.16508},\n",
    "                'Radiomics_Node': {'Fszm_lgze': -0.14667, 'Fcm_corr': 0.48828},\n",
    "                'Radiomics_Primary': {'Fmorph_pca_elongation': 0.04885, 'Fcm_inv_var': -0.08079}\n",
    "}\n",
    "\n",
    "input_ = {\n",
    "    'master': True,\n",
    "    'method': 'master',\n",
    "    'kwargs': {'oropharynx': 'yes',  \n",
    "               'time_col': time_col_name,\n",
    "               'outcome_col': event_col_name,\n",
    "               'coefficients': coefficients, \n",
    "               'organization_ids': [2, 7, 8] # here in the list the organizations ids\n",
    "               }\n",
    "}\n",
    "\n",
    "task = client.task.create(name=\"Querying task\",\n",
    "                               description=\"Send SPARQL queries to fetch required data based on the given feature type\",\n",
    "                               image=\"varshagouthamchand/dcr_sparql_lp\",\n",
    "                               collaboration=1,\n",
    "                               input=input_,\n",
    "                               organizations=[2],\n",
    "                               database='rdf')\n",
    "\n",
    "#print(\"Waiting for results\")\n",
    "task_id = task['id']\n",
    "task_info = client.task.get(task_id)\n",
    "while not task_info.get(\"complete\"):\n",
    "    task_info = client.task.get(task_id, include_results=True)\n",
    "    #print(\"Waiting for results\")\n",
    "    time.sleep(3)\n",
    "\n",
    "print(\"Results are ready!\")\n",
    "\n",
    "result_id = task_info['id']\n",
    "result_info = client.result.list(task=result_id)\n",
    "\n",
    "result = result_info['data'][0]['result']\n",
    "print(result)"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "id": "096b0fa0",
   "metadata": {},
   "source": [],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}