From f69ae3e3553fa2b97227ce4aa66210e292879976 Mon Sep 17 00:00:00 2001 From: afallah Date: Fri, 19 Apr 2024 12:20:26 -0400 Subject: [PATCH 1/8] Fixed critical bug of having duplicate task tokens with tokenizer.py --- odyssey/data/tokenizer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/odyssey/data/tokenizer.py b/odyssey/data/tokenizer.py index e869e36..0072c1c 100644 --- a/odyssey/data/tokenizer.py +++ b/odyssey/data/tokenizer.py @@ -127,7 +127,7 @@ class ConceptTokenizer: Tokenizer object. tokenizer_object: Tokenizer Tokenizer object. - + """ def __init__( @@ -150,6 +150,7 @@ def __init__( self.class_token = class_token self.reg_token = reg_token self.unknown_token = unknown_token + self.task_tokens = ["[MOR_1M]", "[LOS_1W]", "[REA_1M]"] + [ f"[C{i}]" for i in range(0, 5) ] @@ -157,6 +158,7 @@ def __init__( f"c{i}" for i in range(5) ] self.task2token = self.create_task_to_token_dict() + self.special_tokens = ( [ pad_token, @@ -196,13 +198,16 @@ def fit_on_vocab(self, with_tasks: bool = True) -> None: self.token_type_vocab[vocab_type] = vocab if with_tasks: - self.special_tokens += self.task_tokens self.token_type_vocab["task_tokens"] = self.task_tokens # Create the tokenizer dictionary tokens = list(chain.from_iterable(list(self.token_type_vocab.values()))) self.tokenizer_vocab = {token: i for i, token in enumerate(tokens)} + # Update special tokens after tokenizr is created + if with_tasks: + self.special_tokens += self.task_tokens + # Create the tokenizer object self.tokenizer_object = Tokenizer( models.WordPiece( From 21e3f2498dfbd46e56e071f30ccc378b901fad03 Mon Sep 17 00:00:00 2001 From: Amrit Krishnan Date: Fri, 19 Apr 2024 13:35:00 -0400 Subject: [PATCH 2/8] Update docstrings to be consistent with numpy style --- finetune.py | 9 +- odyssey/data/DataProcessor.ipynb | 72 ++++---- odyssey/data/processor.py | 301 ++++++++++++++++++++----------- odyssey/data/tokenizer.py | 4 +- odyssey/evals/TestAnalysis.ipynb | 20 +- odyssey/utils/utils.py | 3 +- 6 files changed, 251 insertions(+), 158 deletions(-) diff --git a/finetune.py b/finetune.py index 3959b76..d2ec290 100644 --- a/finetune.py +++ b/finetune.py @@ -37,7 +37,6 @@ def main( fine_model_config: Dict[str, Any], ) -> None: """Train the model.""" - # Setup environment seed_everything(args.seed) os.environ["CUDA_LAUNCH_BLOCKING"] = "1" @@ -66,7 +65,7 @@ def main( random_state=args.seed, stratify=fine_tune["label"], ) - + else: # Multi label classfication fine_train_ids, _, fine_val_ids, _ = iterative_train_test_split( X=fine_tune["patient_id"].to_numpy().reshape(-1, 1), @@ -113,7 +112,7 @@ def main( balance_guide=None, max_len=args.max_len, ) - + else: train_dataset = FinetuneDataset( data=fine_train, @@ -130,7 +129,7 @@ def main( tokenizer=tokenizer, max_len=args.max_len, ) - + train_loader = DataLoader( train_dataset, batch_size=args.batch_size, @@ -185,7 +184,7 @@ def main( pretrained_model=pretrained_model, **fine_model_config, ) - + elif args.model_type == "cehr_bigbird": pretrained_model = BigBirdPretrain( vocab_size=tokenizer.get_vocab_size(), diff --git a/odyssey/data/DataProcessor.ipynb b/odyssey/data/DataProcessor.ipynb index 0e24f83..b9605c3 100644 --- a/odyssey/data/DataProcessor.ipynb +++ b/odyssey/data/DataProcessor.ipynb @@ -42,9 +42,9 @@ " process_readmission_dataset,\n", " process_multi_dataset,\n", " stratified_train_test_split,\n", - " sample_balanced_subset, \n", + " sample_balanced_subset,\n", " get_pretrain_test_split,\n", - " get_finetune_split\n", + " get_finetune_split,\n", ")\n", "\n", "SEED = 23\n", @@ -77,7 +77,9 @@ "outputs": [], "source": [ "# Process the dataset for length of stay prediction above a threshold\n", - "dataset_2048_los = process_length_of_stay_dataset(dataset_2048.copy(), threshold=7, max_len=MAX_LEN)" + "dataset_2048_los = process_length_of_stay_dataset(\n", + " dataset_2048.copy(), threshold=7, max_len=MAX_LEN\n", + ")" ] }, { @@ -119,7 +121,9 @@ "outputs": [], "source": [ "# Process the dataset for hospital readmission in one month task\n", - "dataset_2048_readmission = process_readmission_dataset(dataset_2048.copy(), max_len=MAX_LEN)" + "dataset_2048_readmission = process_readmission_dataset(\n", + " dataset_2048.copy(), max_len=MAX_LEN\n", + ")" ] }, { @@ -137,7 +141,7 @@ " \"readmission\": dataset_2048_readmission,\n", " \"los\": dataset_2048_los,\n", " },\n", - " max_len=MAX_LEN\n", + " max_len=MAX_LEN,\n", ")" ] }, @@ -174,35 +178,35 @@ "outputs": [], "source": [ "task_config = {\n", - " \"mortality\": {\n", - " \"dataset\": dataset_2048_mortality,\n", - " \"label_col\": \"label_mortality_1month\",\n", - " \"finetune_size\": [250, 500, 1000, 5000, 20000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_mortality.pkl\",\n", - " \"split_mode\": \"single_label_balanced\",\n", - " },\n", - " \"readmission\": {\n", - " \"dataset\": dataset_2048_readmission,\n", - " \"label_col\": \"label_readmission_1month\",\n", - " \"finetune_size\": [250, 1000, 5000, 20000, 60000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_readmission.pkl\",\n", - " \"split_mode\": \"single_label_stratified\",\n", - " },\n", - " \"length_of_stay\": {\n", - " \"dataset\": dataset_2048_los,\n", - " \"label_col\": \"label_los_1week\",\n", - " \"finetune_size\": [250, 1000, 5000, 20000, 50000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_los.pkl\",\n", - " \"split_mode\": \"single_label_balanced\",\n", - " },\n", - " \"condition\": {\n", - " \"dataset\": dataset_2048_condition,\n", - " \"label_col\": \"all_conditions\",\n", - " \"finetune_size\": [50000],\n", - " \"save_path\": \"patient_id_dict/dataset_2048_condition.pkl\",\n", - " \"split_mode\": \"multi_label_stratified\",\n", - " },\n", - " }" + " \"mortality\": {\n", + " \"dataset\": dataset_2048_mortality,\n", + " \"label_col\": \"label_mortality_1month\",\n", + " \"finetune_size\": [250, 500, 1000, 5000, 20000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_mortality.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"readmission\": {\n", + " \"dataset\": dataset_2048_readmission,\n", + " \"label_col\": \"label_readmission_1month\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 60000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_readmission.pkl\",\n", + " \"split_mode\": \"single_label_stratified\",\n", + " },\n", + " \"length_of_stay\": {\n", + " \"dataset\": dataset_2048_los,\n", + " \"label_col\": \"label_los_1week\",\n", + " \"finetune_size\": [250, 1000, 5000, 20000, 50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_los.pkl\",\n", + " \"split_mode\": \"single_label_balanced\",\n", + " },\n", + " \"condition\": {\n", + " \"dataset\": dataset_2048_condition,\n", + " \"label_col\": \"all_conditions\",\n", + " \"finetune_size\": [50000],\n", + " \"save_path\": \"patient_id_dict/dataset_2048_condition.pkl\",\n", + " \"split_mode\": \"multi_label_stratified\",\n", + " },\n", + "}" ] }, { diff --git a/odyssey/data/processor.py b/odyssey/data/processor.py index 4d7c4bc..05e8684 100644 --- a/odyssey/data/processor.py +++ b/odyssey/data/processor.py @@ -1,14 +1,15 @@ -"""Process patient sequences to be usable by model based on task and spilt into trian-test-finetune.""" +"""Process patient sequences based on task and split into train-test-finetune.""" import random -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple import numpy as np import pandas as pd from sklearn.model_selection import train_test_split from skmultilearn.model_selection import iterative_train_test_split -from odyssey.utils.utils import seed_everything, save_object_to_disk +from odyssey.utils.utils import save_object_to_disk, seed_everything + SEED = 23 seed_everything(seed=SEED) @@ -17,30 +18,43 @@ def filter_by_num_visit(dataset: pd.DataFrame, minimum_num_visits: int) -> pd.DataFrame: """Filter the patients based on num_visits threshold. - Args: - dataset (pd.DataFrame): The input dataset. - minimum_num_visits (int): The threshold num_visits + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + minimum_num_visits: int + The threshold number of visits. Returns ------- - pd.DataFrame: The filtered dataset. + pd.DataFrame + The filtered dataset. + """ filtered_dataset = dataset.loc[dataset["num_visits"] >= minimum_num_visits] filtered_dataset.reset_index(drop=True, inplace=True) return filtered_dataset -def filter_by_length_of_stay(dataset: pd.DataFrame, threshold: int = 1, max_len: int = 2048) -> pd.DataFrame: +def filter_by_length_of_stay( + dataset: pd.DataFrame, threshold: int = 1, max_len: int = 2048 +) -> pd.DataFrame: """Filter the patients based on length of stay threshold. - Args: - dataset (pd.DataFrame): The input dataset. - minimum_num_visits (int): The threshold length of stay - max_len (int): The maximum length of the sequence. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + threshold: int + The threshold length of stay. + max_len: int + The maximum length of the sequence. Returns ------- - pd.DataFrame: The filtered dataset. + pd.DataFrame + The filtered dataset. + """ filtered_dataset = dataset.loc[dataset["length_of_stay"] >= threshold] @@ -60,13 +74,18 @@ def filter_by_length_of_stay(dataset: pd.DataFrame, threshold: int = 1, max_len: def get_last_occurence_index(seq: List[str], target: str) -> int: """Return the index of the last occurrence of target in seq. - Args: - seq (List[str]): The input sequence. - target (str): The target string to find. + Parameters + ---------- + seq: List[str] + The input sequence. + target: str + The target token. Returns ------- - int: The index of the last occurrence of target in seq. + int + The index of the last occurrence of target in seq. + """ return len(seq) - (seq[::-1].index(target) + 1) @@ -74,13 +93,18 @@ def get_last_occurence_index(seq: List[str], target: str) -> int: def check_readmission_label(row: pd.Series, max_len: int = 2048) -> int: """Check if the label indicates readmission within one month. - Args: - row (pd.Series): The input row. - max_len (int): The maximum length of the sequence. + Parameters + ---------- + row: pd.Series + The input row. + max_len: int + The maximum length of the sequence. Returns ------- - bool: True if readmission label is present, False otherwise. + int + The readmission label. + """ last_vs_index = row["last_VS_index"] return int( @@ -92,29 +116,42 @@ def check_readmission_label(row: pd.Series, max_len: int = 2048) -> int: def get_length_of_stay(row: pd.Series) -> pd.Series: """Determine the length of a given visit. - Args: - row (pd.Series): The input row. + Parameters + ---------- + row: pd.Series + The input row. Returns ------- - pd.Series: The preprocessed row. + float + The length of stay in days. + """ admission_time = row["last_VS_index"] + 1 discharge_time = row["last_VE_index"] - 1 return (discharge_time - admission_time) / 24 -def get_visit_cutoff_at_threshold(row: pd.Series, threshold: int = 24, max_len: int = 2048) -> int: - """Get the index of the first event token of last visit that occurs after threshold hours. +def get_visit_cutoff_at_threshold( + row: pd.Series, threshold: int = 24, max_len: int = 2048 +) -> int: + """Get the index of the first event token of last visit that cutoff at threshold. - Args: - row (pd.Series): The input row. - threshold (int): The number of hours to consider. - max_len (int): The maximum length of the sequence. + Parameters + ---------- + row: pd.Series + The input row. + threshold: int + The threshold length of stay. + max_len: int + The maximum length of the sequence. Returns ------- - cutoff_index (int): The corrosponding cutoff index. + int + The index of the first event token of last visit that occurs + after threshold hours. + """ last_vs_index = row["last_VS_index"] last_ve_index = row["last_VE_index"] @@ -133,14 +170,20 @@ def process_length_of_stay_dataset( ) -> pd.DataFrame: """Process the length of stay dataset to extract required features. - Args: - dataset (pd.DataFrame): The input dataset. - threshold (int): The threshold length of stay. - max_len (int): The maximum length of the sequence. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + threshold: int + The threshold length of stay. + max_len: int + The maximum length of the sequence. Returns ------- - pd.DataFrame: The processed dataset. + pd.DataFrame + The processed dataset. + """ dataset["last_VS_index"] = dataset[f"event_tokens_{max_len}"].transform( lambda seq: get_last_occurence_index(list(seq), "[VS]"), @@ -165,12 +208,16 @@ def process_length_of_stay_dataset( def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame: """Process the condition dataset to extract required features. - Args: - dataset (pd.DataFrame): The input condition dataset. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. Returns ------- - pd.DataFrame: The processed condition dataset. + pd.DataFrame + The processed dataset. + """ dataset["all_conditions"] = dataset.apply( lambda row: np.concatenate( @@ -186,12 +233,16 @@ def process_condition_dataset(dataset: pd.DataFrame) -> pd.DataFrame: def process_mortality_dataset(dataset: pd.DataFrame) -> pd.DataFrame: """Process the mortality dataset to extract required features. - Args: - dataset (pd.DataFrame): The input mortality dataset. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. Returns ------- - pd.DataFrame: The processed mortality dataset. + pd.DataFrame + The processed dataset. + """ dataset["label_mortality_2weeks"] = ( (dataset["death_after_start"] >= 0) & (dataset["death_after_end"] <= 15) @@ -204,17 +255,24 @@ def process_mortality_dataset(dataset: pd.DataFrame) -> pd.DataFrame: return dataset -def process_readmission_dataset(dataset: pd.DataFrame, max_len: int = 2048) -> pd.DataFrame: +def process_readmission_dataset( + dataset: pd.DataFrame, max_len: int = 2048 +) -> pd.DataFrame: """Process the readmission dataset to extract required features. - Args: - dataset (pd.DataFrame): The input dataset. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + max_len: int + The maximum length of the sequence. Returns ------- - pd.DataFrame: The processed dataset. - """ + pd.DataFrame + The processed dataset. + """ dataset = filter_by_num_visit(dataset.copy(), minimum_num_visits=2) dataset["last_VS_index"] = dataset[f"event_tokens_{max_len}"].transform( @@ -230,23 +288,29 @@ def process_readmission_dataset(dataset: pd.DataFrame, max_len: int = 2048) -> p def process_multi_dataset( - datasets: Dict[str, pd.DataFrame], - max_len: int = 2048, - num_conditions: int = 20, - nan_indicator: int = -1 - ) -> pd.DataFrame: - """ - Process the multi-task dataset by merging the original dataset with the other datasets. - - Args: - datasets (Dict): Dictionary mapping each task to its respective dataframe - max_len (int): The maximum length of the sequence - num_conditions (int): The number of conditions - nan_indicator (int): The indicator for NaN values in dataframe + datasets: Dict[str, pd.DataFrame], + max_len: int = 2048, + num_conditions: int = 20, + nan_indicator: int = -1, +) -> pd.DataFrame: + """Process the multi-task dataset by merging the original dataset with others. + + Parameters + ---------- + datasets: Dict[str, pd.DataFrame] + The input datasets. + max_len: int + The maximum length of the sequence. + num_conditions: int + The number of conditions. + nan_indicator: int + The indicator for NaN values. Returns ------- - pd.DataFrame: The processed multi-task dataset + pd.DataFrame + The processed dataset. + """ # Merging datasets on 'patient_id' multi_dataset = datasets["original"].merge( @@ -335,19 +399,29 @@ def stratified_train_test_split( return_test: Optional[bool] = False, seed: int = SEED, ) -> List[str]: - """Split the given dataset into training and testing sets - using iterative stratification on given multi-label target. - - Args: - dataset (pd.DataFrame): The input dataset. - target (str): The target column for stratification. - test_size (float): The size of the test set. - return_test (bool): Whether to return the test set only. - seed (int): The random seed for reproducibility. + """Split the given dataset into training and testing sets. + + The dataset is stratified using iterative stratification on given multi-label + target. + + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + target: str + The target column for stratification. + test_size: float + The size of the test set. + return_test: Optional[bool] + Whether to return the test set only. + seed: int + The random seed for reproducibility. Returns ------- - List[str]: The patients ids for training and/or testing set. + List[str] + The patient ids for the training or testing set. + """ # Convert all_conditions into a format suitable for multi-label stratification Y = np.array(dataset[target].values.tolist()) @@ -376,34 +450,38 @@ def stratified_train_test_split( if return_test: return X_test - else: - return X_train, X_test + + return X_train, X_test def sample_balanced_subset( - dataset: pd.DataFrame, - target: str, - sample_size: int, - seed: int = SEED + dataset: pd.DataFrame, target: str, sample_size: int, seed: int = SEED ) -> List[str]: """Sample a subset of dataset with balanced target labels. - Args: - dataset (pd.DataFrame): The input dataset. - target (str): The target column for sampling. - sample_size (int): The size of the sample. - seed (int): The random seed for reproducibility. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + target: str + The target column for stratification. + sample_size: int + The size of the sample. + seed: int + The random seed for reproducibility. Returns ------- - List[str]: The patient ids for the sampled set. + List[str] + The patient ids for the balanced sample. + """ # Sampling positive and negative patients - pos_patients = dataset[dataset[target] == True].sample( + pos_patients = dataset[dataset[target] == True].sample( # noqa: E712 n=sample_size // 2, random_state=seed, ) - neg_patients = dataset[dataset[target] == False].sample( + neg_patients = dataset[dataset[target] == False].sample( # noqa: E712 n=sample_size // 2, random_state=seed, ) @@ -421,19 +499,28 @@ def get_pretrain_test_split( dataset: pd.DataFrame, stratify_target: Optional[str] = None, test_size: float = 0.15, - seed: int = SEED + seed: int = SEED, ) -> Tuple[List[str], List[str]]: - """Split dataset into pretrain and test set. Stratify on a given target column if needed. + """Split dataset into pretrain and test set. + + The dataset is stratified on a given target column if needed. - Args: - dataset (pd.DataFrame): The input dataset. - stratify_target (str): The target column for stratification. - test_size (int): The size of the test set. - seed (int): The random seed for reproducibility. + Parameters + ---------- + dataset: pd.DataFrame + The input dataset. + stratify_target: Optional[str] + The target column for stratification. + test_size: float + The size of the test set. + seed: int + The random seed for reproducibility. Returns ------- - Tuple[List[str], List[str]]: The patient ids for the pretrain and test set. + Tuple[List[str], List[str]] + The patient ids for the pretrain and test set. + """ if stratify_target: pretrain_ids, test_ids = stratified_train_test_split( @@ -459,17 +546,25 @@ def get_finetune_split( task: str, patient_ids_dict: Dict[str, Any], ) -> Dict[str, Dict[str, List[str]]]: - """Split the dataset into training and cross-finetuneation sets using k-fold cross-finetuneation - while ensuring balanced label distribution in each fold. Saves the resulting dictionary to disk. + """Split the dataset into training and cross-finetuneation sets + + Using k-fold cross-finetuneation while ensuring balanced label distribution + in each fold, the function saves the resulting dictionary to disk. - Args: - task_config (Any): A dictionray containing the task-specific configuration. - task (str): The task name. Must be one of the keys in the task_config dictionary. - patient_ids_dict (Dict[str, Any]): A dictionary containing the patient ids for each split. + Parameters + ---------- + task_config: Any + The task configuration. + task: str + The task name. + patient_ids_dict: Dict[str, Any] + The dictionary containing patient ids for different splits. Returns ------- - Dict[str, Dict[str, List[str]]]: The updated patient_ids_dict. + Dict[str, Dict[str, List[str]]] + The dictionary containing patient ids for different splits. + """ # Extract task-specific configuration dataset = task_config[task]["dataset"].copy() @@ -484,7 +579,6 @@ def get_finetune_split( # Few-shot finetune patient ids for finetune_num in finetune_sizes: - if split_mode == "single_label_balanced": finetune_ids = sample_balanced_subset( dataset, @@ -492,10 +586,7 @@ def get_finetune_split( sample_size=finetune_num, ) - elif ( - split_mode == "single_label_stratified" - or split_mode == "multi_label_stratified" - ): + elif split_mode in {"single_label_stratified", "multi_label_stratified"}: finetune_ids = stratified_train_test_split( dataset, target=label_col, diff --git a/odyssey/data/tokenizer.py b/odyssey/data/tokenizer.py index 0072c1c..0bca07d 100644 --- a/odyssey/data/tokenizer.py +++ b/odyssey/data/tokenizer.py @@ -50,7 +50,7 @@ def truncate_and_pad( cutoff = min(max_len, len(row[f"event_tokens_{max_len}"])) row[f"event_tokens_{max_len}"] = row[f"event_tokens_{max_len}"][:cutoff] - + row[f"type_tokens_{max_len}"] = np.pad( row[f"type_tokens_{max_len}"][:cutoff], (0, max_len - cutoff), @@ -127,7 +127,7 @@ class ConceptTokenizer: Tokenizer object. tokenizer_object: Tokenizer Tokenizer object. - + """ def __init__( diff --git a/odyssey/evals/TestAnalysis.ipynb b/odyssey/evals/TestAnalysis.ipynb index 3521fdd..87a96b9 100644 --- a/odyssey/evals/TestAnalysis.ipynb +++ b/odyssey/evals/TestAnalysis.ipynb @@ -48,7 +48,7 @@ " valid_scheme = \"few_shot\"\n", " num_finetune_patients = \"all\"\n", " # label_name = \"label_mortality_1month\"\n", - " tasks = ['mortality_1month', 'los_1week', 'c0', 'c1', 'c2']\n", + " tasks = [\"mortality_1month\", \"los_1week\", \"c0\", \"c1\", \"c2\"]\n", "\n", " max_len = 2048\n", " batch_size = 1\n", @@ -100,11 +100,11 @@ ")\n", "\n", "test_dataset = FinetuneMultiDataset(\n", - " data=fine_test,\n", - " tokenizer=tokenizer,\n", - " tasks=config.tasks,\n", - " balance_guide=None,\n", - " max_len=config.max_len,\n", + " data=fine_test,\n", + " tokenizer=tokenizer,\n", + " tasks=config.tasks,\n", + " balance_guide=None,\n", + " max_len=config.max_len,\n", ")\n", "\n", "# train_dataset = FinetuneMultiDataset(\n", @@ -176,8 +176,8 @@ } ], "source": [ - "labels = test_outputs['labels'].cpu().numpy()\n", - "logits = test_outputs['logits'].cpu().numpy()\n", + "labels = test_outputs[\"labels\"].cpu().numpy()\n", + "logits = test_outputs[\"logits\"].cpu().numpy()\n", "probs = torch.sigmoid(torch.tensor(logits[:, 1])).cpu().numpy()\n", "preds = (probs >= 0.5).astype(int)\n", "\n", @@ -211,7 +211,7 @@ "\n", "task_idx = []\n", "for i, task in enumerate(tasks):\n", - " if task == 'los_1week':\n", + " if task == \"los_1week\":\n", " task_idx.append(i)\n", "\n", "\n", @@ -241,7 +241,7 @@ } ], "source": [ - "old_test_outputs = torch.load('test_outputs.pt')\n", + "old_test_outputs = torch.load(\"test_outputs.pt\")\n", "target = 2\n", "\n", "labels = old_test_outputs[\"labels\"][:, target]\n", diff --git a/odyssey/utils/utils.py b/odyssey/utils/utils.py index 62ad861..ad10f1f 100644 --- a/odyssey/utils/utils.py +++ b/odyssey/utils/utils.py @@ -5,9 +5,8 @@ from typing import Any import numpy as np - -import torch import pytorch_lightning as pl +import torch def seed_everything(seed: int) -> None: From 879181c4ac2304fd05dfaf7e914a0101d0c72be1 Mon Sep 17 00:00:00 2001 From: Amrit Krishnan Date: Fri, 19 Apr 2024 14:59:44 -0400 Subject: [PATCH 3/8] Fix type of args in top level scripts --- finetune.py | 2 +- interpret.py | 2 +- odyssey/data/processor.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/finetune.py b/finetune.py index d2ec290..a5a31c1 100644 --- a/finetune.py +++ b/finetune.py @@ -32,7 +32,7 @@ def main( - args: Dict[str, Any], + args: argparse.Namespace, pre_model_config: Dict[str, Any], fine_model_config: Dict[str, Any], ) -> None: diff --git a/interpret.py b/interpret.py index b6be16e..dcb8d54 100644 --- a/interpret.py +++ b/interpret.py @@ -25,7 +25,7 @@ def main( - args: Dict[str, Any], + args: argparse.Namespace, pre_model_config: Dict[str, Any], fine_model_config: Dict[str, Any], ) -> None: diff --git a/odyssey/data/processor.py b/odyssey/data/processor.py index 05e8684..437eed0 100644 --- a/odyssey/data/processor.py +++ b/odyssey/data/processor.py @@ -546,7 +546,7 @@ def get_finetune_split( task: str, patient_ids_dict: Dict[str, Any], ) -> Dict[str, Dict[str, List[str]]]: - """Split the dataset into training and cross-finetuneation sets + """Split the dataset into training and cross-finetuneation sets. Using k-fold cross-finetuneation while ensuring balanced label distribution in each fold, the function saves the resulting dictionary to disk. From a1d350d88aead13a78278805d713efd669582070 Mon Sep 17 00:00:00 2001 From: Adibvafa Fallahpour <90617686+Adibvafa@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:44:50 -0400 Subject: [PATCH 4/8] Create slurm_scripts.md --- slurm_scripts.md | 204 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 slurm_scripts.md diff --git a/slurm_scripts.md b/slurm_scripts.md new file mode 100644 index 0000000..a0477a8 --- /dev/null +++ b/slurm_scripts.md @@ -0,0 +1,204 @@ +# Slurm Job Request Scripts + +## MultiBird - Pretrain +``` +#!/bin/bash +#SBATCH --job-name=multibird_pretrain +#SBATCH --gres=gpu:4 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 24 +#SBATCH --time=23:00:00 +#SBATCH --mem=200G +#SBATCH --output=/h/afallah/odyssey/multibird_a100-%j.out +#SBATCH --error=/h/afallah/odyssey/multibird_a100-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 pretrain.py \ + --model-type cehr_bigbird \ + --exp-name multibird_pretrain \ + --config-dir models/configs \ + --data-dir data/bigbird_data \ + --sequence-file patient_sequences/patient_sequences_2048.parquet \ + --id-file patient_id_dict/dataset_2048_multi.pkl \ + --vocab-dir data/vocab \ + --val-size 0.1 \ + --checkpoint-dir checkpoints/multibird_pretrain +``` + + +## MultiBird - Finetune +``` +#!/bin/bash +#SBATCH --job-name=multibird_finetune +#SBATCH --gres=gpu:4 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 24 +#SBATCH --time=23:59:00 +#SBATCH --mem=200G +#SBATCH --output=/h/afallah/odyssey/multibird_finetune-%j.out +#SBATCH --error=/h/afallah/odyssey/multibird_finetune-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 finetune.py \ + --model-type cehr_bigbird \ + --is-multi-model True \ + --exp-name multibird_finetune \ + --pretrained-path checkpoints/multibird_pretrain/multibird_pretrain/best.ckpt \ + --config-dir odyssey/models/configs \ + --data-dir odyssey/data/bigbird_data \ + --sequence-file patient_sequences_2048_multi.parquet \ + --id-file dataset_2048_multi.pkl \ + --vocab-dir odyssey/data/vocab \ + --val-size 0.15 \ + --valid_scheme few_shot \ + --num_finetune_patients all \ + --problem_type single_label_classification \ + --num_labels 2 \ + --checkpoint-dir checkpoints \ + --test_output_dir test_outputs \ + --tasks "mortality_1month los_1week readmission_1month c0 c1 c2" \ + --balance_guide "mortality_1month=0.5, los_1week=0.5, readmission_1month=0.5, c0=0.5, c1=0.5, c2=0.5" +``` + + +## BigBird - Finetune Mortality +``` +#!/bin/bash +#SBATCH --job-name=bigbird_finetune_mortality +#SBATCH --gres=gpu:2 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 6 +#SBATCH --time=15:00:00 +#SBATCH --mem=32G +#SBATCH --output=/h/afallah/odyssey/bigbird_finetune_mortality-%j.out +#SBATCH --error=/h/afallah/odyssey/bigbird_finetune_mortality-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 finetune.py \ + --model-type cehr_bigbird \ + --exp-name mortality_1month_20000_patients \ + --pretrained-path checkpoints/bigbird_pretrain_with_conditions/pretrain_with_conditions/best-v1.ckpt \ + --label-name label_mortality_1month \ + --config-dir models/configs \ + --data-dir data/bigbird_data \ + --sequence-file patient_sequences/patient_sequences_2048_mortality.parquet \ + --id-file patient_id_dict/dataset_2048_mortality.pkl \ + --vocab-dir data/vocab \ + --val-size 0.1 \ + --valid_scheme few_shot \ + --num_finetune_patients '20000' \ + --problem_type 'single_label_classification' \ + --num_labels 2 \ + --checkpoint-dir checkpoints/bigbird_finetune_with_condition \ + --resume_checkpoint checkpoints/bigbird_finetune_with_condition/mortality_1month_20000_patients/best.ckpt +``` + + +## BigBird - Finetune Condition +``` +#!/bin/bash +#SBATCH --job-name=bigbird_finetune_condition +#SBATCH --gres=gpu:1 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 6 +#SBATCH --time=15:00:00 +#SBATCH --mem=32G +#SBATCH --output=/h/afallah/odyssey/bigbird_finetune_condition-%j.out +#SBATCH --error=/h/afallah/odyssey/bigbird_finetune_condition-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 finetune.py \ + --model-type cehr_bigbird \ + --exp-name condition_50000_patients \ + --pretrained-path checkpoints/bigbird_pretrain_with_conditions/pretrain_with_conditions/best-v1.ckpt \ + --label-name all_conditions \ + --config-dir models/configs \ + --data-dir data/bigbird_data \ + --sequence-file patient_sequences/patient_sequences_2048_condition.parquet \ + --id-file patient_id_dict/dataset_2048_condition.pkl \ + --vocab-dir data/vocab \ + --val-size 0.1 \ + --valid_scheme few_shot \ + --num_finetune_patients '50000' \ + --problem_type 'multi_label_classification' \ + --num_labels 20 \ + --checkpoint-dir checkpoints/bigbird_finetune_with_condition \ + --resume_checkpoint checkpoints/bigbird_finetune_with_condition/condition_50000_patients/best.ckpt +``` + + +## BigBird - Finetune Readmission +``` +#!/bin/bash +#SBATCH --job-name=bigbird_finetune_readmission +#SBATCH --gres=gpu:2 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 6 +#SBATCH --time=15:00:00 +#SBATCH --mem=32G +#SBATCH --output=/h/afallah/odyssey/bigbird_finetune_readmission-%j.out +#SBATCH --error=/h/afallah/odyssey/bigbird_finetune_readmission-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 finetune.py \ + --model-type cehr_bigbird \ + --exp-name readmission_1month_60000_patients \ + --pretrained-path checkpoints/bigbird_pretrain_with_conditions/pretrain_with_conditions/best-v1.ckpt \ + --label-name label_readmission_1month \ + --config-dir models/configs \ + --data-dir data/bigbird_data \ + --sequence-file patient_sequences/patient_sequences_2048_readmission.parquet \ + --id-file patient_id_dict/dataset_2048_readmission.pkl \ + --vocab-dir data/vocab \ + --val-size 0.1 \ + --valid_scheme few_shot \ + --num_finetune_patients '60000' \ + --problem_type 'single_label_classification' \ + --num_labels 2 \ + --checkpoint-dir checkpoints/bigbird_finetune_with_condition +``` From da3eb261bf6d7031bf0fa7abdc8553c493822d04 Mon Sep 17 00:00:00 2001 From: Adibvafa Fallahpour <90617686+Adibvafa@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:46:10 -0400 Subject: [PATCH 5/8] Update slurm_scripts.md --- slurm_scripts.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/slurm_scripts.md b/slurm_scripts.md index a0477a8..7c4d4e4 100644 --- a/slurm_scripts.md +++ b/slurm_scripts.md @@ -202,3 +202,32 @@ stdbuf -oL -eL srun python3 finetune.py \ --num_labels 2 \ --checkpoint-dir checkpoints/bigbird_finetune_with_condition ``` + + +## Bi-LSTM +``` +#!/bin/bash +#SBATCH --job-name=baseline_lstm +#SBATCH --gres=gpu:1 +#SBATCH --qos=normal +#SBATCH --time=6:00:00 +#SBATCH -c 30 +#SBATCH --mem=32G +#SBATCH --ntasks=1 +#SBATCH --output=/h/afallah/odyssey/slurm/baseline_lstm-%j.out +#SBATCH --error=/h/afallah/odyssey/slurm/baseline_lstm-%j.err + +#module --ignore_cache load cuda-11.8 +#module load anaconda/3.10 +#source activate light + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/slurm + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 Bi-LSTM.py +``` From 02a69746ca135a2e1a80b751fc4b7e05ea6182cf Mon Sep 17 00:00:00 2001 From: Adibvafa Fallahpour <90617686+Adibvafa@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:51:08 -0400 Subject: [PATCH 6/8] Update slurm_scripts.md --- slurm_scripts.md | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/slurm_scripts.md b/slurm_scripts.md index 7c4d4e4..ab48416 100644 --- a/slurm_scripts.md +++ b/slurm_scripts.md @@ -79,6 +79,41 @@ stdbuf -oL -eL srun python3 finetune.py \ ``` +## BigBird - Pretrain +``` +#!/bin/bash +#SBATCH --job-name=bigbird_pretrain +#SBATCH --gres=gpu:4 +#SBATCH --qos a100_amritk +#SBATCH -p a100 +#SBATCH -c 24 +#SBATCH --time=23:00:00 +#SBATCH --mem=200G +#SBATCH --output=/h/afallah/odyssey/multibird_a100-%j.out +#SBATCH --error=/h/afallah/odyssey/multibird_a100-%j.err +#SBATCH --no-requeue + +source /h/afallah/light/bin/activate + +cd /h/afallah/odyssey/odyssey + +export CUBLAS_WORKSPACE_CONFIG=:4096:2 +export NCCL_DEBUG=INFO +export PYTHONFAULTHANDLER=1 + +stdbuf -oL -eL srun python3 pretrain.py \ + --model-type cehr_bigbird \ + --exp-name bigbird_pretrain \ + --config-dir models/configs \ + --data-dir data/bigbird_data \ + --sequence-file patient_sequences/patient_sequences_2048.parquet \ + --id-file patient_id_dict/dataset_2048_pretrain.pkl \ + --vocab-dir data/vocab \ + --val-size 0.1 \ + --checkpoint-dir checkpoints/bigbird_pretrain +``` + + ## BigBird - Finetune Mortality ``` #!/bin/bash From e2a1d908d6666099b390aa93bd970ff3347ff08c Mon Sep 17 00:00:00 2001 From: Amrit Krishnan Date: Thu, 25 Apr 2024 16:11:07 -0400 Subject: [PATCH 7/8] Update ruff and fix more issues --- .pre-commit-config.yaml | 25 +- odyssey/data/DataProcessor.ipynb | 89 +- odyssey/evals/CompareAUROC-Poster.ipynb | 20 +- odyssey/evals/TestAnalysis.ipynb | 146 +- odyssey/interp/AttentionVisualization.ipynb | 3010 +---------------- odyssey/interp/grad_attribution.ipynb | 11 +- odyssey/models/baseline/Bi-LSTM.ipynb | 608 +--- odyssey/models/baseline/XGBoost.ipynb | 1207 +------ odyssey/models/cehr_big_bird/playground.ipynb | 1471 +------- odyssey/models/cehr_big_bird/tokenizer.ipynb | 412 +-- 10 files changed, 279 insertions(+), 6720 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fe39392..2aa772d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 # Use the ref you want to point at + rev: v4.6.0 # Use the ref you want to point at hooks: - id: trailing-whitespace - id: check-ast @@ -16,7 +16,7 @@ repos: - id: check-toml - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.3.7' + rev: 'v0.4.2' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] @@ -25,7 +25,7 @@ repos: types_or: [python, jupyter] - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy entry: python3 -m mypy --config-file pyproject.toml @@ -41,3 +41,22 @@ repos: language: system pass_filenames: false always_run: true + + - repo: local + hooks: + - id: nbstripout + name: nbstripout + language: system + entry: python3 -m nbstripout + +ci: + autofix_commit_msg: | + [pre-commit.ci] Add auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: true + autoupdate_branch: '' + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: weekly + skip: [pytest,nbstripout,mypy] + submodules: false diff --git a/odyssey/data/DataProcessor.ipynb b/odyssey/data/DataProcessor.ipynb index b9605c3..a905bce 100644 --- a/odyssey/data/DataProcessor.ipynb +++ b/odyssey/data/DataProcessor.ipynb @@ -3,24 +3,15 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:14:45.546088300Z", - "start_time": "2024-03-13T16:14:43.587090300Z" - }, - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", - "import sys\n", "import pickle\n", - "import random\n", - "from typing import Any, Dict, List, Optional\n", "\n", - "import numpy as np\n", "import pandas as pd\n", "\n", + "\n", "ROOT = \"/h/afallah/odyssey/odyssey\"\n", "DATA_ROOT = f\"{ROOT}/odyssey/data/bigbird_data\"\n", "DATASET = f\"{DATA_ROOT}/patient_sequences/patient_sequences_2048.parquet\"\n", @@ -28,24 +19,16 @@ "\n", "os.chdir(ROOT)\n", "\n", - "from odyssey.utils.utils import seed_everything\n", "from odyssey.data.processor import (\n", - " filter_by_num_visit,\n", - " filter_by_length_of_stay,\n", - " get_last_occurence_index,\n", - " check_readmission_label,\n", - " get_length_of_stay,\n", - " get_visit_cutoff_at_threshold,\n", - " process_length_of_stay_dataset,\n", + " get_finetune_split,\n", " process_condition_dataset,\n", + " process_length_of_stay_dataset,\n", " process_mortality_dataset,\n", - " process_readmission_dataset,\n", " process_multi_dataset,\n", - " stratified_train_test_split,\n", - " sample_balanced_subset,\n", - " get_pretrain_test_split,\n", - " get_finetune_split,\n", + " process_readmission_dataset,\n", ")\n", + "from odyssey.utils.utils import seed_everything\n", + "\n", "\n", "SEED = 23\n", "seed_everything(seed=SEED)" @@ -54,13 +37,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:12.321718600Z", - "start_time": "2024-03-13T16:14:45.553089800Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Load complete dataset\n", @@ -95,13 +72,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:16.075719400Z", - "start_time": "2024-03-13T16:15:12.335721100Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Process the dataset for mortality in two weeks or one month task\n", @@ -111,13 +82,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:47.326996100Z", - "start_time": "2024-03-13T16:15:16.094719300Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Process the dataset for hospital readmission in one month task\n", @@ -212,17 +177,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T16:15:51.800996200Z", - "start_time": "2024-03-13T16:15:50.494996100Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Get finetune split\n", - "for task in task_config.keys():\n", + "for task in task_config:\n", " patient_ids_dict = get_finetune_split(\n", " task_config=task_config,\n", " task=task,\n", @@ -233,13 +192,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T14:14:10.181184300Z", - "start_time": "2024-03-13T14:13:39.154567400Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "dataset_2048_mortality.to_parquet(\n", @@ -365,9 +318,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Assuming dataset.event_tokens is your DataFrame column\n", @@ -383,9 +334,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# len(patient_ids_dict['group3']['cv'])\n", @@ -402,9 +351,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "##### DEAD ZONE | DO NOT ENTER #####\n", @@ -424,9 +371,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# dataset_2048_readmission = dataset_2048.loc[dataset_2048['num_visits'] > 1]\n", diff --git a/odyssey/evals/CompareAUROC-Poster.ipynb b/odyssey/evals/CompareAUROC-Poster.ipynb index 9e92592..f1facf9 100644 --- a/odyssey/evals/CompareAUROC-Poster.ipynb +++ b/odyssey/evals/CompareAUROC-Poster.ipynb @@ -3,9 +3,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", @@ -19,9 +17,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Import dependencies and define useful constants\n", @@ -51,9 +47,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Load predictions, labels, and probabilities of different models\n", @@ -81,9 +75,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Plot ROC Curve for XGBoost, Bi-LSTM, and Transformer\n", @@ -103,9 +95,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Plot Information\n", diff --git a/odyssey/evals/TestAnalysis.ipynb b/odyssey/evals/TestAnalysis.ipynb index 87a96b9..f2ffdc4 100644 --- a/odyssey/evals/TestAnalysis.ipynb +++ b/odyssey/evals/TestAnalysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19,21 +19,19 @@ " recall_score,\n", " roc_auc_score,\n", ")\n", - "from transformers import utils\n", "\n", - "from tqdm import tqdm\n", "\n", "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", + "from odyssey.data.dataset import FinetuneMultiDataset\n", "from odyssey.data.tokenizer import ConceptTokenizer\n", - "from odyssey.models.model_utils import load_finetune_data\n", - "from odyssey.data.dataset import FinetuneMultiDataset" + "from odyssey.models.model_utils import load_finetune_data" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -57,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -120,40 +118,18 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "20600" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "tokenizer.get_vocab_size()" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['labels', 'logits'])" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "test_outputs = torch.load(config.model_path, map_location=config.device)\n", "test_outputs.keys()" @@ -161,20 +137,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([0, 0, 0, ..., 0, 0, 0])" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "labels = test_outputs[\"labels\"].cpu().numpy()\n", "logits = test_outputs[\"logits\"].cpu().numpy()\n", @@ -186,26 +151,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Balanced Accuracy': 0.8043450887917278,\n", - " 'F1 Score': 0.7121043864519712,\n", - " 'Precision': 0.6445533358462119,\n", - " 'Recall': 0.7954721662273221,\n", - " 'AUROC': 0.8889347846976665,\n", - " 'Average Precision Score': 0.5738031917760823,\n", - " 'AUC-PR': 0.7505522277674918}" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Tasks we have are: tasks = ['mortality_1month', 'los_1week', 'c0', 'c1', 'c2']\n", "\n", @@ -220,26 +168,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Balanced Accuracy': 0.8220779574282759,\n", - " 'F1 Score': 0.7388243277631691,\n", - " 'Precision': 0.7633654688869412,\n", - " 'Recall': 0.7158119658119658,\n", - " 'AUROC': 0.9255784114401215,\n", - " 'Average Precision Score': 0.6157970244149283,\n", - " 'AUC-PR': 0.7742741610984505}" - ] - }, - "execution_count": 24, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "old_test_outputs = torch.load(\"test_outputs.pt\")\n", "target = 2\n", @@ -252,60 +183,27 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(0)" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "preds.sum()" ] }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor(34., dtype=torch.float64)" - ] - }, - "execution_count": 69, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "labels.sum()" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([0, 0, 0, ..., 0, 0, 0], dtype=torch.int32)" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "preds" ] diff --git a/odyssey/interp/AttentionVisualization.ipynb b/odyssey/interp/AttentionVisualization.ipynb index 2a40a8f..3539168 100644 --- a/odyssey/interp/AttentionVisualization.ipynb +++ b/odyssey/interp/AttentionVisualization.ipynb @@ -2,27 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:47.301592Z", - "start_time": "2024-03-15T15:39:47.293763Z" - }, - "collapsed": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "'\\nFile: AttentionVisualization.ipynb\\n---------------------------------\\nVisualize the attention layers of transformer models for interpretability.\\n'" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "\"\"\"\n", "File: AttentionVisualization.ipynb\n", @@ -33,15 +16,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "e24876a0c6020df2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:49:16.880004Z", - "start_time": "2024-03-15T15:49:16.866092Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -63,23 +40,17 @@ "\n", "from odyssey.data.dataset import FinetuneDataset\n", "from odyssey.data.tokenizer import ConceptTokenizer\n", - "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes\n", "from odyssey.models.model_utils import (\n", " load_finetune_data,\n", - ")" + ")\n", + "from odyssey.models.prediction import load_finetuned_model, predict_patient_outcomes" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "dac09785d1fdc0cc", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:54.875569Z", - "start_time": "2024-03-15T15:39:54.852027Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "class args:\n", @@ -105,15 +76,9 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "6b5e21258ced06d2", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:39:54.921944Z", - "start_time": "2024-03-15T15:39:54.877586Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer = ConceptTokenizer(data_dir=args.vocab_dir)\n", @@ -122,153 +87,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "8bf63dc4242a163e", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:40:00.629410Z", - "start_time": "2024-03-15T15:39:54.929550Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "BigBirdFinetune(\n", - " (model): BigBirdForSequenceClassification(\n", - " (bert): BigBirdModel(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BigBirdEncoder(\n", - " (layer): ModuleList(\n", - " (0-5): 6 x BigBirdLayer(\n", - " (attention): BigBirdAttention(\n", - " (self): BigBirdBlockSparseAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " )\n", - " (output): BigBirdSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BigBirdIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): NewGELUActivation()\n", - " )\n", - " (output): BigBirdOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " (classifier): BigBirdClassificationHead(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (out_proj): Linear(in_features=768, out_features=2, bias=True)\n", - " )\n", - " )\n", - " (pretrained_model): BigBirdPretrain(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (model): BigBirdForMaskedLM(\n", - " (bert): BigBirdModel(\n", - " (embeddings): BigBirdEmbeddingsForCEHR(\n", - " (word_embeddings): Embedding(20592, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(2048, 768)\n", - " (token_type_embeddings): Embedding(9, 768)\n", - " (time_embeddings): TimeEmbeddingLayer()\n", - " (age_embeddings): TimeEmbeddingLayer()\n", - " (visit_segment_embeddings): VisitEmbedding(\n", - " (embedding): Embedding(3, 768)\n", - " )\n", - " (scale_back_concat_layer): Linear(in_features=832, out_features=768, bias=True)\n", - " (tanh): Tanh()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BigBirdEncoder(\n", - " (layer): ModuleList(\n", - " (0-5): 6 x BigBirdLayer(\n", - " (attention): BigBirdAttention(\n", - " (self): BigBirdBlockSparseAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " )\n", - " (output): BigBirdSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BigBirdIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " (intermediate_act_fn): NewGELUActivation()\n", - " )\n", - " (output): BigBirdOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " (cls): BigBirdOnlyMLMHead(\n", - " (predictions): BigBirdLMPredictionHead(\n", - " (transform): BigBirdPredictionHeadTransform(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (transform_act_fn): NewGELUActivation()\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " )\n", - " (decoder): Linear(in_features=768, out_features=20592, bias=True)\n", - " )\n", - " )\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "model = load_finetuned_model(args.model_path, tokenizer)\n", "model" @@ -276,15 +98,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "45bb295710f64bc0", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:23.262096Z", - "start_time": "2024-03-15T15:52:23.252429Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "fine_tune, fine_test = load_finetune_data(\n", @@ -301,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "6c2f9f1f", "metadata": {}, "outputs": [], @@ -320,34 +136,10 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "ead6f3658dda5274", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:24.058853Z", - "start_time": "2024-03-15T15:52:24.027868Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'concept_ids': tensor([[ 5, 3, 18065, ..., 0, 0, 0]]),\n", - " 'type_ids': tensor([[1, 2, 6, ..., 0, 0, 0]]),\n", - " 'ages': tensor([[ 0, 67, 67, ..., 0, 0, 0]]),\n", - " 'time_stamps': tensor([[ 0, 5773, 5773, ..., 0, 0, 0]]),\n", - " 'visit_orders': tensor([[0, 1, 1, ..., 0, 0, 0]]),\n", - " 'visit_segments': tensor([[0, 2, 2, ..., 0, 0, 0]]),\n", - " 'labels': tensor([0]),\n", - " 'attention_mask': tensor([[1, 1, 1, ..., 0, 0, 0]])}" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "patient = next(iter(test_loader))\n", "patient" @@ -355,586 +147,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "b025c033efc9baab", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:25.797704Z", - "start_time": "2024-03-15T15:52:25.476580Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "SequenceClassifierOutput(loss=tensor(0.1007, device='cuda:0', grad_fn=), logits=tensor([[ 1.1001, -1.1447]], device='cuda:0', grad_fn=), hidden_states=(tensor([[[-0.6737, -0.4033, -1.0631, ..., -0.0571, -0.5688, -2.8107],\n", - " [-0.2583, 1.4081, 0.0247, ..., 0.9723, -0.1994, -0.6298],\n", - " [-0.3349, -0.7147, -0.2382, ..., 0.6685, -2.0221, 0.3603],\n", - " ...,\n", - " [-1.1487, -0.3301, -1.2087, ..., -1.4240, 0.6589, -3.1762],\n", - " [-0.5165, 0.4454, -1.0153, ..., -2.3917, -0.0144, -3.3358],\n", - " [-0.1893, -0.1348, 0.2863, ..., -2.4897, 0.7176, -2.4138]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-9.1614e-01, 6.7605e-01, 6.2693e-03, ..., -3.1620e-01,\n", - " -4.6282e-01, -2.0648e+00],\n", - " [-6.0621e-01, 8.6801e-01, 2.6213e-01, ..., 3.4474e-01,\n", - " 1.3363e-01, -7.1667e-01],\n", - " [-7.6982e-01, 2.3106e-03, -5.9440e-01, ..., -4.7166e-01,\n", - " -1.8369e+00, 1.3552e-01],\n", - " ...,\n", - " [-8.6433e-01, 7.3538e-01, -3.3437e-01, ..., -1.2050e+00,\n", - " 1.3254e+00, -2.4991e+00],\n", - " [-7.3595e-01, 1.2425e+00, 5.3519e-03, ..., -1.8429e+00,\n", - " 9.7859e-01, -2.8092e+00],\n", - " [-5.8273e-01, 8.9714e-01, 8.5189e-01, ..., -1.7958e+00,\n", - " 1.6784e+00, -2.0120e+00]]], device='cuda:0',\n", - " grad_fn=), tensor([[[-0.5963, 1.0668, 0.6960, ..., -0.5670, -1.8491, -0.7775],\n", - " [-0.8661, 0.9432, 1.3912, ..., -0.1141, -0.0171, -0.0536],\n", - " [-0.4142, 0.1496, 0.0763, ..., -1.3690, -2.2675, 0.8359],\n", - " ...,\n", - " [-0.5217, 1.3100, 0.3550, ..., -0.8422, 0.7881, -2.2435],\n", - " [-0.6279, 1.6811, 0.6428, ..., -1.1621, 0.4885, -2.3759],\n", - " [-0.5811, 1.5146, 1.1337, ..., -1.0819, 1.0710, -2.0434]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.4564, 0.8573, 0.4777, ..., -1.0131, -1.2389, -0.7232],\n", - " [-0.7503, 0.8020, 2.2105, ..., 0.3817, -0.5328, -1.1600],\n", - " [-0.2785, 0.3193, 0.7508, ..., -0.8207, -2.1549, -0.0479],\n", - " ...,\n", - " [-0.5749, 0.9360, 0.3542, ..., -1.2162, 0.5259, -1.9892],\n", - " [-0.8316, 1.2188, 0.4210, ..., -1.5618, 0.3291, -2.1654],\n", - " [-0.7359, 1.2650, 0.9190, ..., -1.4486, 0.7956, -1.8708]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.6359, 1.1296, 0.3645, ..., -1.0155, -1.4793, -0.0216],\n", - " [-0.2409, 0.9339, 1.1703, ..., 1.1403, -1.0845, -0.8589],\n", - " [-0.2531, 0.2159, 0.2502, ..., -0.1171, -2.6449, -0.1982],\n", - " ...,\n", - " [-0.4492, 0.9411, 0.7911, ..., -0.9499, 0.3742, -1.3986],\n", - " [-0.7147, 1.2283, 0.7656, ..., -1.0737, 0.1806, -1.6471],\n", - " [-0.6436, 1.0595, 1.1752, ..., -1.1340, 0.4983, -1.3118]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.0605, 0.4757, 1.2138, ..., -1.2107, -0.7777, 0.4597],\n", - " [-0.0597, 0.6793, 0.6923, ..., 0.5595, -0.6465, -0.4138],\n", - " [-0.2956, 0.0119, 0.0849, ..., -0.4024, -1.7548, 0.2857],\n", - " ...,\n", - " [-0.0730, 1.2001, 0.8387, ..., -1.5022, 0.3392, -0.9774],\n", - " [-0.2355, 1.2640, 0.8785, ..., -1.6132, 0.2851, -1.2461],\n", - " [-0.2264, 1.1501, 1.2094, ..., -1.7025, 0.5599, -0.9631]]],\n", - " device='cuda:0', grad_fn=), tensor([[[-0.4529, 0.7745, 0.9677, ..., -0.7731, 0.3017, 0.6295],\n", - " [-0.2527, 0.7169, 0.3931, ..., 0.3904, -0.2026, 0.2191],\n", - " [-0.7347, 0.0648, 0.1225, ..., -0.4706, -0.9406, 0.7601],\n", - " ...,\n", - " [-0.1587, 1.8421, 0.7164, ..., -1.1803, 0.3151, -1.0949],\n", - " [-0.4175, 1.9637, 0.7340, ..., -1.2465, 0.3226, -1.4348],\n", - " [-0.4056, 1.7997, 1.0610, ..., -1.3635, 0.5409, -1.1068]]],\n", - " device='cuda:0', grad_fn=)), attentions=(tensor([[[[9.2084e-01, 3.4662e-05, 1.6030e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0152e-04, 1.0245e-03, 5.0193e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1340e-05, 3.4795e-04, 1.1249e-01, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.7594e-01, 6.5887e-04, 2.1206e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4787e-01, 1.3629e-03, 7.0707e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9477e-01, 1.2923e-03, 4.1574e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.2569e-05, 3.1873e-03, 1.4330e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0942e-04, 7.4366e-03, 1.0470e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.2275e-07, 8.8712e-03, 2.0909e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.3820e-04, 5.4235e-03, 3.9131e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1625e-05, 6.0716e-03, 9.2111e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0256e-05, 6.8238e-03, 1.3608e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.1552e-05, 3.2614e-03, 5.1249e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0069e-02, 1.7049e-03, 7.4966e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6985e-04, 7.9826e-05, 1.0739e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.2891e-05, 2.2444e-03, 8.0701e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6524e-04, 2.5353e-03, 1.1610e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.4759e-05, 3.7951e-03, 9.3934e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[9.5957e-07, 1.8916e-05, 1.7706e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4017e-03, 6.3007e-04, 8.2404e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7625e-05, 2.4848e-05, 1.9090e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.4380e-05, 1.2709e-04, 2.4316e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0306e-05, 1.8831e-04, 2.8286e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8297e-05, 1.9117e-04, 2.4082e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.7399e-03, 1.2051e-03, 9.5375e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7488e-03, 2.7973e-03, 2.2183e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6722e-03, 7.9718e-04, 2.1421e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6540e-03, 2.6361e-03, 1.2919e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7157e-03, 2.6202e-03, 1.2555e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6167e-03, 1.7538e-03, 1.1150e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.5929e-03, 3.2053e-03, 2.2594e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3881e-03, 1.7186e-03, 4.7067e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7379e-03, 3.9178e-03, 5.7301e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.5827e-03, 2.9827e-03, 2.1182e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3835e-03, 3.9488e-03, 2.3409e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.8265e-03, 3.3581e-03, 2.8842e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[3.4325e-03, 3.6487e-03, 2.1270e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.6434e-03, 2.1695e-02, 6.8258e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.8946e-03, 5.1628e-02, 9.1872e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.2476e-03, 3.5812e-03, 3.8917e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.5847e-03, 5.8187e-03, 4.7901e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7512e-03, 7.3790e-03, 5.0303e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.9891e-05, 4.4143e-06, 7.2098e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.8795e-03, 2.2040e-02, 1.3027e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.2756e-02, 1.8536e-02, 1.8427e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0230e-04, 1.0431e-04, 3.6400e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0183e-04, 1.0241e-04, 4.4847e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2058e-04, 2.5562e-04, 6.5510e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.7265e-03, 6.2247e-06, 3.0965e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.5288e-05, 1.7324e-02, 6.0294e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.6073e-04, 1.7897e-03, 2.7479e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5983e-03, 2.8453e-04, 4.3261e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3822e-03, 2.4970e-04, 7.7571e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9758e-03, 2.7208e-04, 6.9291e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.2307e-04, 1.6154e-07, 1.1893e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5087e-04, 7.7591e-02, 4.4232e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.9669e-04, 7.1647e-03, 1.2542e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6157e-03, 1.9115e-04, 8.7496e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5545e-03, 6.9437e-04, 2.2060e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5639e-03, 4.7817e-04, 1.0144e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[9.0331e-02, 2.0968e-04, 1.5666e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.2050e-05, 4.9400e-03, 2.9039e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.8098e-04, 7.9459e-03, 1.0302e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.0900e-02, 5.0021e-03, 3.6893e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7062e-02, 5.0122e-03, 5.1272e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.2155e-02, 4.1310e-03, 4.8125e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.2098e-04, 4.5407e-08, 1.1781e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.4916e-05, 1.8071e-03, 2.9574e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9490e-03, 3.8689e-03, 1.6880e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.2944e-03, 1.2684e-05, 3.9500e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5202e-03, 9.5840e-06, 3.9704e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4998e-03, 1.0348e-05, 4.0015e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[9.0947e-03, 1.7104e-04, 7.2082e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0905e-03, 8.2947e-03, 4.7806e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2774e-02, 4.1130e-03, 1.0427e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.4360e-03, 2.5052e-03, 3.5916e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5451e-03, 2.0411e-03, 4.1614e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0480e-02, 1.5095e-03, 3.9783e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.0953e-04, 4.8485e-07, 1.1093e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7053e-06, 7.0000e-03, 1.3357e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2624e-03, 4.3490e-03, 8.2277e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.3735e-03, 9.6488e-05, 1.6069e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6108e-03, 6.0472e-05, 1.0010e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.4534e-03, 9.8078e-05, 1.4517e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1851e-03, 3.6355e-09, 4.5110e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1155e-05, 1.3445e-03, 3.1936e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.9026e-03, 3.4348e-04, 3.4485e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.4469e-03, 1.2559e-05, 5.1731e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.1810e-03, 6.3667e-06, 5.4800e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0399e-03, 6.1432e-06, 4.1856e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[1.9304e-02, 3.4070e-05, 1.5798e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7734e-05, 1.5453e-02, 6.9360e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.5957e-04, 3.6940e-03, 5.8250e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6903e-03, 1.8801e-03, 4.4015e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2805e-03, 8.8522e-04, 5.1555e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4006e-03, 1.2400e-03, 5.7507e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1703e-04, 4.1161e-05, 7.9415e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.8420e-04, 7.7026e-03, 7.9656e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5139e-03, 2.4419e-03, 2.6276e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.7707e-03, 6.9963e-04, 1.6709e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0923e-03, 7.9106e-04, 2.5133e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4068e-03, 2.6501e-04, 1.1517e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.0330e-05, 1.8324e-06, 1.0670e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.0295e-04, 6.8225e-03, 4.1048e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6344e-03, 2.1405e-03, 1.9475e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0302e-03, 2.2379e-05, 8.3888e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0699e-03, 3.1538e-05, 9.3927e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1386e-03, 4.4208e-05, 1.2655e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[1.0968e-03, 1.1158e-05, 2.8874e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.5882e-04, 1.9889e-02, 2.9087e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0075e-03, 2.4018e-03, 5.6122e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2534e-03, 5.0390e-05, 2.8556e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.8193e-04, 5.5810e-05, 3.2226e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8834e-03, 1.9344e-04, 8.2757e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.1752e-04, 1.0848e-03, 5.2472e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3445e-05, 3.1222e-02, 7.3892e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.9519e-04, 2.3304e-02, 4.0696e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [7.4697e-04, 2.2354e-04, 5.6568e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.7230e-04, 4.3862e-04, 9.0070e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0728e-04, 3.7051e-04, 8.7622e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.4030e-04, 1.0463e-03, 1.2372e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3409e-04, 2.6108e-02, 1.0480e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3558e-03, 4.1270e-03, 3.1541e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.7033e-04, 1.3328e-04, 4.4750e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1584e-03, 5.6131e-04, 1.9963e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3833e-03, 1.7700e-04, 1.2080e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.0260e-04, 1.0307e-03, 5.4334e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5736e-04, 4.4732e-03, 3.1108e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0399e-03, 2.5841e-03, 3.4921e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2584e-03, 3.3595e-04, 1.9949e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4056e-03, 7.4162e-04, 3.0422e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4776e-03, 3.4124e-04, 2.3692e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.3945e-05, 2.8038e-06, 7.7244e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1178e-03, 5.9071e-01, 3.8243e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1613e-03, 9.0446e-03, 1.3656e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.1137e-06, 8.0224e-07, 2.9675e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1845e-05, 4.3952e-06, 1.5539e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0935e-05, 7.2282e-06, 1.6119e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[7.7009e-03, 9.1021e-04, 4.4472e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0289e-04, 1.2443e-02, 8.9628e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.7645e-04, 5.7296e-03, 3.5397e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.8769e-04, 7.9107e-06, 6.3012e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.2969e-04, 4.7156e-05, 2.6878e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.0934e-04, 3.7384e-05, 2.2858e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[6.7381e-04, 1.1823e-02, 7.4050e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.4863e-04, 4.0161e-02, 3.6514e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6004e-03, 1.9051e-02, 5.0602e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.5047e-03, 1.9051e-03, 1.5601e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.6917e-03, 3.4615e-03, 1.6968e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.7805e-03, 3.3552e-03, 2.2134e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[3.1542e-04, 1.2458e-05, 3.5918e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5336e-03, 2.6060e-03, 9.9650e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3570e-03, 4.4454e-04, 2.1211e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.8644e-05, 1.1354e-06, 2.5000e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0465e-05, 9.6188e-07, 2.3662e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5490e-05, 1.7402e-06, 4.0543e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.4482e-03, 4.3810e-07, 2.4999e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1627e-04, 5.7318e-02, 2.5052e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0123e-03, 1.0834e-03, 3.6955e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2926e-03, 2.7957e-07, 6.0144e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3540e-03, 2.3200e-07, 7.3312e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2753e-03, 5.4792e-07, 9.2235e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[1.8663e-05, 6.3300e-07, 1.4228e-06, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2953e-03, 1.2528e-02, 3.5547e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9098e-03, 1.1120e-03, 1.3087e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.1750e-04, 1.4822e-05, 2.3723e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.7225e-05, 8.3810e-06, 1.6070e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6516e-04, 2.6572e-05, 4.2221e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.5335e-05, 4.0905e-04, 2.3216e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7303e-04, 5.1438e-02, 3.8481e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3572e-04, 1.6108e-02, 4.7815e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6582e-03, 9.6089e-04, 3.1044e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1655e-03, 1.2231e-03, 3.3471e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5900e-03, 1.6183e-03, 5.6334e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.3592e-03, 9.1240e-05, 1.9648e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.9554e-04, 3.4671e-02, 1.2307e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1695e-03, 3.3091e-03, 4.3312e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5615e-04, 4.8380e-07, 7.2619e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2036e-03, 3.6028e-06, 3.0705e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2138e-03, 3.5186e-06, 2.6554e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=), tensor([[[[4.0090e-03, 6.7263e-04, 7.5136e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9121e-04, 1.2857e-02, 3.9387e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5724e-03, 4.5150e-03, 2.8897e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5598e-03, 1.4921e-04, 2.2929e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.6446e-03, 1.8594e-04, 2.9366e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.2847e-03, 2.3700e-04, 3.6638e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.9915e-02, 4.0394e-05, 5.0094e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.3356e-04, 3.0291e-03, 1.2489e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3452e-03, 8.8680e-04, 1.3824e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0090e-01, 1.4063e-05, 1.0046e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1156e-01, 1.0029e-05, 8.7870e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5568e-02, 1.6524e-05, 1.0813e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.8186e-02, 5.9815e-06, 5.0354e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1144e-04, 3.3203e-03, 1.0907e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1076e-03, 7.5293e-04, 1.7252e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2483e-01, 2.2352e-07, 2.1154e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1819e-01, 1.1773e-07, 1.4315e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0731e-01, 2.1415e-07, 1.9841e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[8.9774e-03, 2.6941e-04, 3.7395e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7763e-04, 4.7386e-03, 2.2903e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2463e-03, 2.2257e-03, 1.9620e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.1848e-02, 8.8093e-05, 1.7620e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3670e-02, 8.8600e-05, 1.7867e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2773e-02, 1.0601e-04, 2.1455e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.7395e-02, 2.0985e-04, 9.8349e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3505e-04, 3.5468e-03, 1.4784e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9124e-03, 1.9014e-03, 2.2491e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.7175e-02, 1.4688e-05, 9.3223e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9626e-02, 9.9576e-06, 9.4240e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6579e-02, 1.2380e-05, 9.0343e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.5388e-03, 4.6041e-05, 9.6215e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3881e-05, 1.0036e-02, 3.3659e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.2542e-04, 6.3507e-03, 5.8799e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.5240e-04, 1.0778e-08, 7.3738e-08, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5682e-04, 1.5447e-08, 1.0267e-07, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6646e-04, 3.5608e-08, 2.2134e-07, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], device='cuda:0', grad_fn=)))" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "output = predict_patient_outcomes(patient, model)\n", "output" @@ -942,15 +158,9 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "e1aa8309b2650820", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T15:52:33.578704Z", - "start_time": "2024-03-15T15:52:33.571290Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokens = tokenizer.decode(patient[\"concept_ids\"].squeeze(0).cpu().numpy()).split(\" \")\n", @@ -962,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "e9203f59", "metadata": {}, "outputs": [], @@ -980,2158 +190,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Token [CLS] with Token 50893_0: Attention Value 0.154\n", - "Token [CLS] with Token 51251_1: Attention Value 0.026\n", - "Token [CLS] with Token 51279_3: Attention Value 0.018\n", - "Token [CLS] with Token 50983_0: Attention Value 0.014\n", - "Token [CLS] with Token 51257_0: Attention Value 0.013\n", - "Token [CLS] with Token 52069_0: Attention Value 0.012\n", - "Token [CLS] with Token 50902_0: Attention Value 0.011\n", - "Token [CLS] with Token 51277_1: Attention Value 0.010\n", - "Token [CLS] with Token 50878_4: Attention Value 0.009\n", - "Token [CLS] with Token 00641607825: Attention Value 0.009\n", - "Token [CLS] with Token 00641607825: Attention Value 0.009\n", - "Token [CLS] with Token 51133_2: Attention Value 0.009\n", - "Token [CLS] with Token [CLS]: Attention Value 0.008\n", - "Token [CLS] with Token 51255_1: Attention Value 0.008\n", - "Token [CLS] with Token 61958040101: Attention Value 0.008\n" - ] - }, - { - "data": { - "application/vnd.plotly.v1+json": { - "config": { - "plotlyServerURL": "https://plot.ly" - }, - "data": [ - { - "colorscale": [ - [ - 0, - "rgb(255,255,217)" - ], - [ - 0.125, - "rgb(237,248,177)" - ], - [ - 0.25, - "rgb(199,233,180)" - ], - [ - 0.375, - "rgb(127,205,187)" - ], - [ - 0.5, - "rgb(65,182,196)" - ], - [ - 0.625, - "rgb(29,145,192)" - ], - [ - 0.75, - "rgb(34,94,168)" - ], - [ - 0.875, - "rgb(37,52,148)" - ], - [ - 1, - "rgb(8,29,88)" - ] - ], - "hoverinfo": "text", - "hoverongaps": false, - "text": [ - [ - "Token [CLS] with Token [CLS]:Attention Value 0.008", - "Token [CLS] with Token [VS]:Attention Value 0.000", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00006494300:Attention Value 0.000", - "Token [CLS] with Token 00904272561:Attention Value 0.001", - "Token [CLS] with Token 61958040101:Attention Value 0.004", - "Token [CLS] with Token 00487950101:Attention Value 0.002", - "Token [CLS] with Token 00904198261:Attention Value 0.001", - "Token [CLS] with Token 00173071920:Attention Value 0.007", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 60505011300:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 61958040101:Attention Value 0.008", - "Token [CLS] with Token 00597008717:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 42292000120:Attention Value 0.002", - "Token [CLS] with Token 00904639161:Attention Value 0.001", - "Token [CLS] with Token 00338004904:Attention Value 0.005", - "Token [CLS] with Token 00472030680:Attention Value 0.003", - "Token [CLS] with Token 00143989701:Attention Value 0.001", - "Token [CLS] with Token 00641607825:Attention Value 0.009", - "Token [CLS] with Token 00338004904:Attention Value 0.004", - "Token [CLS] with Token 51079096620:Attention Value 0.001", - "Token [CLS] with Token 00409128331:Attention Value 0.002", - "Token [CLS] with Token 63323031461:Attention Value 0.000", - "Token [CLS] with Token 00009513503:Attention Value 0.001", - "Token [CLS] with Token 00056051030:Attention Value 0.004", - "Token [CLS] with Token 66758016013:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00338004904:Attention Value 0.003", - "Token [CLS] with Token 00071015892:Attention Value 0.004", - "Token [CLS] with Token 00338004904:Attention Value 0.006", - "Token [CLS] with Token 11523726808:Attention Value 0.001", - "Token [CLS] with Token 00904198861:Attention Value 0.000", - "Token [CLS] with Token 00071015892:Attention Value 0.003", - "Token [CLS] with Token 00487020101:Attention Value 0.002", - "Token [CLS] with Token 63739035810:Attention Value 0.001", - "Token [CLS] with Token 00904635361:Attention Value 0.003", - "Token [CLS] with Token 76439034310:Attention Value 0.005", - "Token [CLS] with Token 57896042101:Attention Value 0.003", - "Token [CLS] with Token 00056051030:Attention Value 0.005", - "Token [CLS] with Token 42292000120:Attention Value 0.002", - "Token [CLS] with Token 51079054720:Attention Value 0.004", - "Token [CLS] with Token 00904224461:Attention Value 0.000", - "Token [CLS] with Token 51079098320:Attention Value 0.002", - "Token [CLS] with Token 61958060101:Attention Value 0.003", - "Token [CLS] with Token 51079098320:Attention Value 0.001", - "Token [CLS] with Token 00641607825:Attention Value 0.009", - "Token [CLS] with Token 49281041688:Attention Value 0.005", - "Token [CLS] with Token 00904516561:Attention Value 0.002", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 00173068224:Attention Value 0.005", - "Token [CLS] with Token 00121048910:Attention Value 0.001", - "Token [CLS] with Token 63323026201:Attention Value 0.000", - "Token [CLS] with Token 57896045208:Attention Value 0.004", - "Token [CLS] with Token 63323031461:Attention Value 0.001", - "Token [CLS] with Token 00487020101:Attention Value 0.002", - "Token [CLS] with Token 52074_3:Attention Value 0.000", - "Token [CLS] with Token 52073_2:Attention Value 0.000", - "Token [CLS] with Token 52069_2:Attention Value 0.000", - "Token [CLS] with Token 51301_1:Attention Value 0.000", - "Token [CLS] with Token 51279_3:Attention Value 0.001", - "Token [CLS] with Token 51277_1:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.000", - "Token [CLS] with Token 51256_1:Attention Value 0.001", - "Token [CLS] with Token 51254_4:Attention Value 0.001", - "Token [CLS] with Token 51250_2:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_3:Attention Value 0.000", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_3:Attention Value 0.002", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_2:Attention Value 0.001", - "Token [CLS] with Token 51133_3:Attention Value 0.001", - "Token [CLS] with Token 52075_2:Attention Value 0.001", - "Token [CLS] with Token 52074_2:Attention Value 0.001", - "Token [CLS] with Token 52073_2:Attention Value 0.001", - "Token [CLS] with Token 52069_1:Attention Value 0.001", - "Token [CLS] with Token 51301_1:Attention Value 0.001", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.001", - "Token [CLS] with Token 51265_1:Attention Value 0.000", - "Token [CLS] with Token 51256_3:Attention Value 0.001", - "Token [CLS] with Token 51254_3:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.000", - "Token [CLS] with Token 51222_3:Attention Value 0.001", - "Token [CLS] with Token 51221_3:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_1:Attention Value 0.001", - "Token [CLS] with Token 51133_1:Attention Value 0.001", - "Token [CLS] with Token 51006_0:Attention Value 0.001", - "Token [CLS] with Token 50983_0:Attention Value 0.002", - "Token [CLS] with Token 50971_0:Attention Value 0.001", - "Token [CLS] with Token 50970_1:Attention Value 0.001", - "Token [CLS] with Token 50960_2:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.002", - "Token [CLS] with Token 50902_1:Attention Value 0.001", - "Token [CLS] with Token 50893_0:Attention Value 0.003", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50878_3:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 50863_1:Attention Value 0.001", - "Token [CLS] with Token 50861_3:Attention Value 0.001", - "Token [CLS] with Token 51516_1:Attention Value 0.001", - "Token [CLS] with Token 51498_4:Attention Value 0.001", - "Token [CLS] with Token 51493_2:Attention Value 0.000", - "Token [CLS] with Token 51492_1:Attention Value 0.001", - "Token [CLS] with Token 51491_2:Attention Value 0.001", - "Token [CLS] with Token 51009_0:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_0:Attention Value 0.004", - "Token [CLS] with Token 50971_0:Attention Value 0.002", - "Token [CLS] with Token 50970_1:Attention Value 0.001", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.002", - "Token [CLS] with Token 50893_0:Attention Value 0.003", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.002", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_1:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_3:Attention Value 0.002", - "Token [CLS] with Token 51221_3:Attention Value 0.001", - "Token [CLS] with Token 50813_3:Attention Value 0.002", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.002", - "Token [CLS] with Token 50971_1:Attention Value 0.002", - "Token [CLS] with Token 50970_3:Attention Value 0.001", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50911_0:Attention Value 0.001", - "Token [CLS] with Token 50902_1:Attention Value 0.001", - "Token [CLS] with Token 50893_0:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 51301_3:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_4:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51516_3:Attention Value 0.001", - "Token [CLS] with Token 51498_4:Attention Value 0.001", - "Token [CLS] with Token 51493_2:Attention Value 0.001", - "Token [CLS] with Token 51491_2:Attention Value 0.001", - "Token [CLS] with Token 51482_2:Attention Value 0.001", - "Token [CLS] with Token 50813_4:Attention Value 0.003", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.003", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50970_1:Attention Value 0.002", - "Token [CLS] with Token 50960_0:Attention Value 0.001", - "Token [CLS] with Token 50931_4:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.005", - "Token [CLS] with Token 50902_2:Attention Value 0.003", - "Token [CLS] with Token 50893_2:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 52135_2:Attention Value 0.002", - "Token [CLS] with Token 52075_4:Attention Value 0.002", - "Token [CLS] with Token 52074_4:Attention Value 0.002", - "Token [CLS] with Token 52073_3:Attention Value 0.001", - "Token [CLS] with Token 52069_3:Attention Value 0.002", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_4:Attention Value 0.003", - "Token [CLS] with Token 51277_0:Attention Value 0.004", - "Token [CLS] with Token 51265_4:Attention Value 0.001", - "Token [CLS] with Token 51256_3:Attention Value 0.003", - "Token [CLS] with Token 51254_3:Attention Value 0.002", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.001", - "Token [CLS] with Token 51146_2:Attention Value 0.001", - "Token [CLS] with Token 51133_2:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50983_1:Attention Value 0.002", - "Token [CLS] with Token 50971_3:Attention Value 0.002", - "Token [CLS] with Token 50970_3:Attention Value 0.000", - "Token [CLS] with Token 50960_3:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_2:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.002", - "Token [CLS] with Token 50893_3:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.000", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51006_1:Attention Value 0.001", - "Token [CLS] with Token 50983_1:Attention Value 0.001", - "Token [CLS] with Token 50971_1:Attention Value 0.001", - "Token [CLS] with Token 50970_2:Attention Value 0.000", - "Token [CLS] with Token 50960_3:Attention Value 0.001", - "Token [CLS] with Token 50931_2:Attention Value 0.000", - "Token [CLS] with Token 50912_2:Attention Value 0.001", - "Token [CLS] with Token 50902_2:Attention Value 0.001", - "Token [CLS] with Token 50893_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51277_0:Attention Value 0.001", - "Token [CLS] with Token 51279_4:Attention Value 0.001", - "Token [CLS] with Token 51301_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 50861_1:Attention Value 0.001", - "Token [CLS] with Token 50863_1:Attention Value 0.001", - "Token [CLS] with Token 50868_4:Attention Value 0.001", - "Token [CLS] with Token 50878_0:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.001", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50902_3:Attention Value 0.003", - "Token [CLS] with Token 50912_2:Attention Value 0.002", - "Token [CLS] with Token 50931_1:Attention Value 0.001", - "Token [CLS] with Token 50971_2:Attention Value 0.002", - "Token [CLS] with Token 50983_3:Attention Value 0.002", - "Token [CLS] with Token 51006_1:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.003", - "Token [CLS] with Token 50893_1:Attention Value 0.003", - "Token [CLS] with Token 50902_2:Attention Value 0.005", - "Token [CLS] with Token 50912_2:Attention Value 0.004", - "Token [CLS] with Token 50931_2:Attention Value 0.002", - "Token [CLS] with Token 50960_2:Attention Value 0.002", - "Token [CLS] with Token 50970_2:Attention Value 0.002", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50983_1:Attention Value 0.004", - "Token [CLS] with Token 51006_1:Attention Value 0.004", - "Token [CLS] with Token 51009_2:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.003", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51476_0:Attention Value 0.002", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 52135_4:Attention Value 0.003", - "Token [CLS] with Token 51492_1:Attention Value 0.002", - "Token [CLS] with Token 51476_0:Attention Value 0.002", - "Token [CLS] with Token 51275_0:Attention Value 0.002", - "Token [CLS] with Token 51274_0:Attention Value 0.002", - "Token [CLS] with Token 51237_1:Attention Value 0.001", - "Token [CLS] with Token 51006_0:Attention Value 0.003", - "Token [CLS] with Token 50983_0:Attention Value 0.005", - "Token [CLS] with Token 50971_1:Attention Value 0.003", - "Token [CLS] with Token 50970_0:Attention Value 0.002", - "Token [CLS] with Token 50960_4:Attention Value 0.002", - "Token [CLS] with Token 50931_2:Attention Value 0.001", - "Token [CLS] with Token 50912_3:Attention Value 0.005", - "Token [CLS] with Token 50902_0:Attention Value 0.003", - "Token [CLS] with Token 50893_2:Attention Value 0.002", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.004", - "Token [CLS] with Token 50878_4:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50863_2:Attention Value 0.001", - "Token [CLS] with Token 50861_4:Attention Value 0.002", - "Token [CLS] with Token 52075_3:Attention Value 0.002", - "Token [CLS] with Token 52074_3:Attention Value 0.002", - "Token [CLS] with Token 52073_2:Attention Value 0.001", - "Token [CLS] with Token 52069_3:Attention Value 0.002", - "Token [CLS] with Token 51301_4:Attention Value 0.002", - "Token [CLS] with Token 51279_3:Attention Value 0.002", - "Token [CLS] with Token 51277_1:Attention Value 0.002", - "Token [CLS] with Token 51265_3:Attention Value 0.001", - "Token [CLS] with Token 51256_3:Attention Value 0.003", - "Token [CLS] with Token 51254_2:Attention Value 0.001", - "Token [CLS] with Token 51250_3:Attention Value 0.001", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_1:Attention Value 0.001", - "Token [CLS] with Token 51222_4:Attention Value 0.002", - "Token [CLS] with Token 51221_4:Attention Value 0.001", - "Token [CLS] with Token 51200_2:Attention Value 0.002", - "Token [CLS] with Token 51146_2:Attention Value 0.002", - "Token [CLS] with Token 51133_3:Attention Value 0.001", - "Token [CLS] with Token 51275_0:Attention Value 0.002", - "Token [CLS] with Token 51274_1:Attention Value 0.003", - "Token [CLS] with Token 51237_2:Attention Value 0.002", - "Token [CLS] with Token 51006_0:Attention Value 0.003", - "Token [CLS] with Token 50983_0:Attention Value 0.004", - "Token [CLS] with Token 50976_2:Attention Value 0.002", - "Token [CLS] with Token 50971_0:Attention Value 0.002", - "Token [CLS] with Token 50970_2:Attention Value 0.001", - "Token [CLS] with Token 50960_2:Attention Value 0.001", - "Token [CLS] with Token 50951_1:Attention Value 0.001", - "Token [CLS] with Token 50950_2:Attention Value 0.002", - "Token [CLS] with Token 50949_1:Attention Value 0.004", - "Token [CLS] with Token 50931_3:Attention Value 0.001", - "Token [CLS] with Token 50930_3:Attention Value 0.002", - "Token [CLS] with Token 50912_3:Attention Value 0.003", - "Token [CLS] with Token 50902_0:Attention Value 0.002", - "Token [CLS] with Token 50893_0:Attention Value 0.004", - "Token [CLS] with Token 50885_1:Attention Value 0.001", - "Token [CLS] with Token 50882_0:Attention Value 0.002", - "Token [CLS] with Token 50878_4:Attention Value 0.002", - "Token [CLS] with Token 50868_4:Attention Value 0.002", - "Token [CLS] with Token 50863_2:Attention Value 0.001", - "Token [CLS] with Token 50862_3:Attention Value 0.001", - "Token [CLS] with Token 50861_3:Attention Value 0.001", - "Token [CLS] with Token 50853_1:Attention Value 0.006", - "Token [CLS] with Token 52075_2:Attention Value 0.004", - "Token [CLS] with Token 52074_3:Attention Value 0.002", - "Token [CLS] with Token 52073_3:Attention Value 0.001", - "Token [CLS] with Token 52069_0:Attention Value 0.012", - "Token [CLS] with Token 51301_1:Attention Value 0.006", - "Token [CLS] with Token 51279_3:Attention Value 0.018", - "Token [CLS] with Token 51277_1:Attention Value 0.010", - "Token [CLS] with Token 51265_2:Attention Value 0.001", - "Token [CLS] with Token 51257_0:Attention Value 0.013", - "Token [CLS] with Token 51256_1:Attention Value 0.003", - "Token [CLS] with Token 51255_1:Attention Value 0.008", - "Token [CLS] with Token 51254_4:Attention Value 0.003", - "Token [CLS] with Token 51251_1:Attention Value 0.026", - "Token [CLS] with Token 51250_2:Attention Value 0.003", - "Token [CLS] with Token 51248_3:Attention Value 0.001", - "Token [CLS] with Token 51244_2:Attention Value 0.002", - "Token [CLS] with Token 51222_4:Attention Value 0.003", - "Token [CLS] with Token 51221_3:Attention Value 0.002", - "Token [CLS] with Token 51200_3:Attention Value 0.003", - "Token [CLS] with Token 51146_0:Attention Value 0.003", - "Token [CLS] with Token 51144_2:Attention Value 0.002", - "Token [CLS] with Token 51143_0:Attention Value 0.002", - "Token [CLS] with Token 51133_2:Attention Value 0.009", - "Token [CLS] with Token 51006_0:Attention Value 0.007", - "Token [CLS] with Token 50983_0:Attention Value 0.014", - "Token [CLS] with Token 50971_0:Attention Value 0.006", - "Token [CLS] with Token 50970_1:Attention Value 0.002", - "Token [CLS] with Token 50960_2:Attention Value 0.004", - "Token [CLS] with Token 50950_2:Attention Value 0.005", - "Token [CLS] with Token 50931_2:Attention Value 0.003", - "Token [CLS] with Token 50912_3:Attention Value 0.008", - "Token [CLS] with Token 50902_0:Attention Value 0.011", - "Token [CLS] with Token 50893_0:Attention Value 0.154", - "Token [CLS] with Token 50885_1:Attention Value 0.002", - "Token [CLS] with Token 50882_0:Attention Value 0.004", - "Token [CLS] with Token 50878_4:Attention Value 0.009", - "Token [CLS] with Token 50868_4:Attention Value 0.007", - "Token [CLS] with Token 50863_2:Attention Value 0.005", - "Token [CLS] with Token 50861_3:Attention Value 0.002", - "Token [CLS] with Token 52075_2:Attention Value 0.004", - "Token [CLS] with Token [VE]:Attention Value 0.006", - "Token [CLS] with Token [REG]:Attention Value 0.003" - ] - ], - "type": "heatmap", - "x": [ - "[CLS]", - "[VS]", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "30", - "31", - "32", - "33", - "34", - "35", - "36", - "37", - "38", - "39", - "40", - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "51", - "52", - "53", - "54", - "55", - "56", - "57", - "58", - "59", - "60", - "61", - "62", - "63", - "64", - "65", - "66", - "67", - "68", - "69", - "70", - "71", - "72", - "73", - "74", - "75", - "76", - "77", - "78", - "79", - "80", - "81", - "82", - "83", - "84", - "85", - "86", - "87", - "88", - "89", - "90", - "91", - "92", - "93", - "94", - "95", - "96", - "97", - "98", - "99", - "100", - "101", - "102", - "103", - "104", - "105", - "106", - "107", - "108", - "109", - "110", - "111", - "112", - "113", - "114", - "115", - "116", - "117", - "118", - "119", - "120", - "121", - "122", - "123", - "124", - "125", - "126", - "127", - "128", - "129", - "130", - "131", - "132", - "133", - "134", - "135", - "136", - "137", - "138", - "139", - "140", - "141", - "142", - "143", - "144", - "145", - "146", - "147", - "148", - "149", - "150", - "151", - "152", - "153", - "154", - "155", - "156", - "157", - "158", - "159", - "160", - "161", - "162", - "163", - "164", - "165", - "166", - "167", - "168", - "169", - "170", - "171", - "172", - "173", - "174", - "175", - "176", - "177", - "178", - "179", - "180", - "181", - "182", - "183", - "184", - "185", - "186", - "187", - "188", - "189", - "190", - "191", - "192", - "193", - "194", - "195", - "196", - "197", - "198", - "199", - "200", - "201", - "202", - "203", - "204", - "205", - "206", - "207", - "208", - "209", - "210", - "211", - "212", - "213", - "214", - "215", - "216", - "217", - "218", - "219", - "220", - "221", - "222", - "223", - "224", - "225", - "226", - "227", - "228", - "229", - "230", - "231", - "232", - "233", - "234", - "235", - "236", - "237", - "238", - "239", - "240", - "241", - "242", - "243", - "244", - "245", - "246", - "247", - "248", - "249", - "250", - "251", - "252", - "253", - "254", - "255", - "256", - "257", - "258", - "259", - "260", - "261", - "262", - "263", - "264", - "265", - "266", - "267", - "268", - "269", - "270", - "271", - "272", - "273", - "274", - "275", - "276", - "277", - "278", - "279", - "280", - "281", - "282", - "283", - "284", - "285", - "286", - "287", - "288", - "289", - "290", - "291", - "292", - "293", - "294", - "295", - "296", - "297", - "298", - "299", - "300", - "301", - "302", - "303", - "304", - "305", - "306", - "307", - "308", - "309", - "310", - "311", - "312", - "313", - "314", - "315", - "316", - "317", - "318", - "319", - "320", - "321", - "322", - "323", - "324", - "325", - "326", - "327", - "328", - "329", - "330", - "331", - "332", - "333", - "334", - "335", - "336", - "337", - "338", - "339", - "340", - "341", - "342", - "343", - "344", - "345", - "346", - "347", - "348", - "349", - "350", - "351", - "352", - "353", - "354", - "355", - "356", - "357", - "358", - "359", - "360", - "361", - "362", - "363", - "364", - "365", - "366", - "367", - "368", - "369", - "370", - "371", - "372", - "373", - "374", - "375", - "[VE]", - "[REG]" - ], - "y": [ - "[CLS]" - ], - "z": [ - [ - 0.008451767265796661, - 0.00047753742546774447, - 0.001319802482612431, - 0.002615319797769189, - 0.0005728917894884944, - 0.0004854540165979415, - 0.0007460298365913332, - 0.004369142930954695, - 0.0021512836683541536, - 0.0010651483898982406, - 0.006857313681393862, - 0.0009411114733666182, - 0.001146065187640488, - 0.003502822481095791, - 0.008377399295568466, - 0.00369816436432302, - 0.0036342681851238017, - 0.001818674267269671, - 0.0011760754277929664, - 0.005098414141684771, - 0.003414525417611003, - 0.0005102005670778453, - 0.009316006675362589, - 0.0036000509280711412, - 0.0010369595838710666, - 0.0019885266665369272, - 0.0004615136131178588, - 0.0008582020527683198, - 0.004428019281476736, - 0.0036287240218371153, - 0.003203341970220208, - 0.003022358985617757, - 0.004367131274193525, - 0.005930684972554445, - 0.0006912555545568466, - 0.0003829135384876281, - 0.002646421082317829, - 0.001659494242630899, - 0.0008906491566449404, - 0.0034986960235983133, - 0.004725936334580183, - 0.003188258036971092, - 0.004636996425688267, - 0.0017536324448883531, - 0.003574026981368661, - 0.0004167505830992013, - 0.001605302095413208, - 0.0034259206149727106, - 0.0010352873941883445, - 0.008988478220999241, - 0.004783918149769306, - 0.0020494854543358088, - 0.0008181784651242197, - 0.005045545753091574, - 0.0008356395992450416, - 0.0003578341274987906, - 0.004442144185304642, - 0.0007386466022580862, - 0.0015157737070694566, - 0.0003436319238971919, - 0.00029586421442218125, - 0.0004052982258144766, - 0.0004121463280171156, - 0.0006444680620916188, - 0.0006272507016547024, - 0.0003953258565161377, - 0.0005427289288491011, - 0.000674175564199686, - 0.0005934844375588, - 0.0005067584570497274, - 0.00048668019007891417, - 0.0006641843356192112, - 0.001736009493470192, - 0.0006450659711845219, - 0.0006109003443270922, - 0.0014823578530922532, - 0.0007514643366448581, - 0.0010008058743551371, - 0.0005770500865764916, - 0.0009993098210543394, - 0.0013827277580276132, - 0.0017192121595144272, - 0.0011489095631986856, - 0.0004682957660406828, - 0.0008927697781473398, - 0.000683685124386102, - 0.0006812589126639068, - 0.0005206714267842472, - 0.0004224051663186401, - 0.0008084981818683445, - 0.001108424854464829, - 0.0009732283069752156, - 0.0007592075853608549, - 0.0005552212824113667, - 0.00127204111777246, - 0.002062402432784438, - 0.00112050655297935, - 0.0008525322191417217, - 0.0008413223549723625, - 0.000753440021071583, - 0.0019989104475826025, - 0.0010396834695711732, - 0.00325021636672318, - 0.0006715958588756621, - 0.0012460827128961682, - 0.000999152078293264, - 0.0009655823814682662, - 0.0005084593431092799, - 0.0005497524398379028, - 0.0006103277555666864, - 0.001034797285683453, - 0.0004598199157044291, - 0.0008794588502496481, - 0.000696350762154907, - 0.001486221794039011, - 0.0017987970495596528, - 0.003589141182601452, - 0.001990983495488763, - 0.0010251256171613932, - 0.001364864525385201, - 0.0008277010056190193, - 0.0025306493043899536, - 0.0021720887161791325, - 0.003414546838030219, - 0.0019456190057098863, - 0.0013443160569295287, - 0.001797817298211157, - 0.0022308507468551397, - 0.0018264207756146789, - 0.0006515602581202984, - 0.0015126297948881984, - 0.0009584605577401816, - 0.0015974263660609722, - 0.0012428780319169164, - 0.0018666594987735152, - 0.0018526790663599968, - 0.001926859957166016, - 0.001790812355466187, - 0.0006914885598234832, - 0.0009147145319730044, - 0.0006848545162938535, - 0.003264268860220909, - 0.001066222321242094, - 0.0014806516701355577, - 0.0019578184001147747, - 0.001972360536456108, - 0.001908112782984972, - 0.0017941169207915664, - 0.002000534441322088, - 0.00232343259267509, - 0.0006864610477350652, - 0.0015883835731074214, - 0.001452704076655209, - 0.0015260959044098854, - 0.001217573182657361, - 0.0007630472537130117, - 0.0009102648473344744, - 0.0010583768598735332, - 0.0010495075257495046, - 0.0013496936298906803, - 0.002790031023323536, - 0.0016460781916975975, - 0.001800714642740786, - 0.001588907209224999, - 0.0006608786643482745, - 0.0015428924234583974, - 0.00130733463447541, - 0.0019498993642628193, - 0.0011935298098251224, - 0.002081812592223286, - 0.0027750423178076744, - 0.002682143822312355, - 0.0017559939296916127, - 0.0014839059440419078, - 0.0011782203800976276, - 0.005421904847025871, - 0.0033844260033220053, - 0.0019421830074861648, - 0.002427612664178014, - 0.0021077985875308514, - 0.002597493352368474, - 0.0019147455459460616, - 0.0022990747820585966, - 0.00196700356900692, - 0.0013677203096449375, - 0.0017343414947390556, - 0.0022338901180773973, - 0.002618071623146534, - 0.003806268097832799, - 0.0009631214197725058, - 0.0029177824035286903, - 0.0015531842363998294, - 0.00118330551777035, - 0.0009476374834775924, - 0.0007008272805251181, - 0.0012339174281805754, - 0.0011241122847422955, - 0.0009876210242509842, - 0.0008716397569514811, - 0.001008612453006208, - 0.0015318739460781217, - 0.0016837789444252849, - 0.0015550237149000168, - 0.00042292437865398824, - 0.0012427183100953698, - 0.0006267700809985399, - 0.0025690405163913965, - 0.002414754591882229, - 0.0008665476925671101, - 0.0007410055841319263, - 0.0011087892344221473, - 0.0014391514705494046, - 0.001120170229114592, - 0.0013088533887639642, - 0.0004123076796531677, - 0.0007661065901629627, - 0.0005386698176153004, - 0.0007872642017900944, - 0.0005083107971586287, - 0.0009654506575316192, - 0.0009281352977268398, - 0.0006618788465857506, - 0.0004174182831775397, - 0.0006613184814341366, - 0.0003955806605517864, - 0.0011703699128702285, - 0.0007652677595615387, - 0.0009616908500902356, - 0.000610732939094305, - 0.0005696555017493665, - 0.000935644842684269, - 0.0005050115869380534, - 0.0008104036678560078, - 0.0005484815337695181, - 0.0005282653728500009, - 0.0006690899026580155, - 0.0006202083895914257, - 0.0011858937796205282, - 0.0006323098205029964, - 0.0012637152103707194, - 0.0010147326393052936, - 0.001434296485967934, - 0.0010879833716899157, - 0.0007398917223326862, - 0.0006289978045970201, - 0.0010291151702404022, - 0.0007981405942700803, - 0.0012479289434850216, - 0.000667006301227957, - 0.0027725808322429657, - 0.002342016203328967, - 0.0006513544940389693, - 0.0018907733028754592, - 0.002159562660381198, - 0.0017471732571721077, - 0.0015442147850990295, - 0.003038904396817088, - 0.003148901043459773, - 0.005466174334287643, - 0.004158675670623779, - 0.001521154772490263, - 0.002499047899618745, - 0.0023495368659496307, - 0.0034855001140385866, - 0.004092938732355833, - 0.003764112712815404, - 0.0024812589399516582, - 0.0029548786114901304, - 0.00236957217566669, - 0.0012237809132784605, - 0.001522903912700713, - 0.0026057965587824583, - 0.0028801767621189356, - 0.0020971631165593863, - 0.0017077381489798429, - 0.002488095546141267, - 0.001942558796145022, - 0.0012060959124937654, - 0.0032189686316996813, - 0.004897533915936947, - 0.002546509262174368, - 0.002436422742903233, - 0.0016161665553227067, - 0.0008353215525858104, - 0.004716587718576193, - 0.003007955150678754, - 0.001906886580400169, - 0.0010407179361209271, - 0.0036604618653655057, - 0.0019616391509771347, - 0.001880012801848352, - 0.0010364892659708858, - 0.0020254242699593306, - 0.001567527768202126, - 0.0018129348754882812, - 0.0011344858212396502, - 0.001787062268704176, - 0.002030221512541175, - 0.002426884835585952, - 0.0018259655917063355, - 0.0006956890574656427, - 0.0026017874479293823, - 0.0008315623854286969, - 0.0014645435148850083, - 0.0009511765674687922, - 0.0009218405466526748, - 0.0015906983753666282, - 0.0014132362557575109, - 0.0017343055224046111, - 0.0019996073096990585, - 0.0013999902876093984, - 0.0021138533484190702, - 0.002869958057999611, - 0.0016324784373864532, - 0.003336323192343116, - 0.004306184593588114, - 0.0022455891594290733, - 0.0018202642677351832, - 0.0014412502059713006, - 0.0014982676366344094, - 0.0010419118916615844, - 0.0016191820614039898, - 0.003920895978808403, - 0.0013921178178861735, - 0.0016874541761353612, - 0.003467828966677189, - 0.0024901803117245436, - 0.003774721873924136, - 0.0008989623747766018, - 0.0018224921077489853, - 0.001697454135864973, - 0.002475267043337226, - 0.001093029393814504, - 0.0014057605294510722, - 0.0009937405120581388, - 0.006045807618647814, - 0.003723071655258536, - 0.0024961901362985373, - 0.0010668792529031634, - 0.012105301953852177, - 0.006213821936398745, - 0.018133636564016346, - 0.010110520757734776, - 0.001374133862555027, - 0.013326204381883144, - 0.002680495148524642, - 0.008382327854633331, - 0.0030930601060390472, - 0.025627313181757927, - 0.0031410318333655596, - 0.001220939448103309, - 0.0015257663326337934, - 0.0025776990223675966, - 0.0020443866960704327, - 0.003117623971775174, - 0.0028798922430723906, - 0.001909662154503167, - 0.0018645658856257796, - 0.00894069205969572, - 0.006883753929287195, - 0.014036208391189575, - 0.0055046118795871735, - 0.0024932369124144316, - 0.0035899977665394545, - 0.004715373273938894, - 0.002608843147754669, - 0.007800393272191286, - 0.011004121042788029, - 0.15411949157714844, - 0.0017875274643301964, - 0.0042082518339157104, - 0.009378299117088318, - 0.007050686050206423, - 0.005348358768969774, - 0.0016888940008357167, - 0.004475145135074854, - 0.006452843081206083, - 0.0025523456279188395 - ] - ] - } - ], - "layout": { - "annotations": [ - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[CLS]", - "textangle": -90, - "x": 0, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[VS]", - "textangle": -90, - "x": 1, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[VE]", - "textangle": -90, - "x": 376, - "xref": "x", - "y": 0.5, - "yref": "paper" - }, - { - "bgcolor": "red", - "font": { - "color": "black", - "size": 10 - }, - "opacity": 0.8, - "showarrow": false, - "text": "[REG]", - "textangle": -90, - "x": 377, - "xref": "x", - "y": 0.5, - "yref": "paper" - } - ], - "template": { - "data": { - "bar": [ - { - "error_x": { - "color": "#2a3f5f" - }, - "error_y": { - "color": "#2a3f5f" - }, - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "bar" - } - ], - "barpolar": [ - { - "marker": { - "line": { - "color": "#E5ECF6", - "width": 0.5 - }, - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "barpolar" - } - ], - "carpet": [ - { - "aaxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "baxis": { - "endlinecolor": "#2a3f5f", - "gridcolor": "white", - "linecolor": "white", - "minorgridcolor": "white", - "startlinecolor": "#2a3f5f" - }, - "type": "carpet" - } - ], - "choropleth": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "choropleth" - } - ], - "contour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "contour" - } - ], - "contourcarpet": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "contourcarpet" - } - ], - "heatmap": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmap" - } - ], - "heatmapgl": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "heatmapgl" - } - ], - "histogram": [ - { - "marker": { - "pattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - } - }, - "type": "histogram" - } - ], - "histogram2d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2d" - } - ], - "histogram2dcontour": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "histogram2dcontour" - } - ], - "mesh3d": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "type": "mesh3d" - } - ], - "parcoords": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "parcoords" - } - ], - "pie": [ - { - "automargin": true, - "type": "pie" - } - ], - "scatter": [ - { - "fillpattern": { - "fillmode": "overlay", - "size": 10, - "solidity": 0.2 - }, - "type": "scatter" - } - ], - "scatter3d": [ - { - "line": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatter3d" - } - ], - "scattercarpet": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattercarpet" - } - ], - "scattergeo": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergeo" - } - ], - "scattergl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattergl" - } - ], - "scattermapbox": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scattermapbox" - } - ], - "scatterpolar": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolar" - } - ], - "scatterpolargl": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterpolargl" - } - ], - "scatterternary": [ - { - "marker": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "type": "scatterternary" - } - ], - "surface": [ - { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - }, - "colorscale": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "type": "surface" - } - ], - "table": [ - { - "cells": { - "fill": { - "color": "#EBF0F8" - }, - "line": { - "color": "white" - } - }, - "header": { - "fill": { - "color": "#C8D4E3" - }, - "line": { - "color": "white" - } - }, - "type": "table" - } - ] - }, - "layout": { - "annotationdefaults": { - "arrowcolor": "#2a3f5f", - "arrowhead": 0, - "arrowwidth": 1 - }, - "autotypenumbers": "strict", - "coloraxis": { - "colorbar": { - "outlinewidth": 0, - "ticks": "" - } - }, - "colorscale": { - "diverging": [ - [ - 0, - "#8e0152" - ], - [ - 0.1, - "#c51b7d" - ], - [ - 0.2, - "#de77ae" - ], - [ - 0.3, - "#f1b6da" - ], - [ - 0.4, - "#fde0ef" - ], - [ - 0.5, - "#f7f7f7" - ], - [ - 0.6, - "#e6f5d0" - ], - [ - 0.7, - "#b8e186" - ], - [ - 0.8, - "#7fbc41" - ], - [ - 0.9, - "#4d9221" - ], - [ - 1, - "#276419" - ] - ], - "sequential": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ], - "sequentialminus": [ - [ - 0, - "#0d0887" - ], - [ - 0.1111111111111111, - "#46039f" - ], - [ - 0.2222222222222222, - "#7201a8" - ], - [ - 0.3333333333333333, - "#9c179e" - ], - [ - 0.4444444444444444, - "#bd3786" - ], - [ - 0.5555555555555556, - "#d8576b" - ], - [ - 0.6666666666666666, - "#ed7953" - ], - [ - 0.7777777777777778, - "#fb9f3a" - ], - [ - 0.8888888888888888, - "#fdca26" - ], - [ - 1, - "#f0f921" - ] - ] - }, - "colorway": [ - "#636efa", - "#EF553B", - "#00cc96", - "#ab63fa", - "#FFA15A", - "#19d3f3", - "#FF6692", - "#B6E880", - "#FF97FF", - "#FECB52" - ], - "font": { - "color": "#2a3f5f" - }, - "geo": { - "bgcolor": "white", - "lakecolor": "white", - "landcolor": "#E5ECF6", - "showlakes": true, - "showland": true, - "subunitcolor": "white" - }, - "hoverlabel": { - "align": "left" - }, - "hovermode": "closest", - "mapbox": { - "style": "light" - }, - "paper_bgcolor": "white", - "plot_bgcolor": "#E5ECF6", - "polar": { - "angularaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "radialaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "scene": { - "xaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "yaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - }, - "zaxis": { - "backgroundcolor": "#E5ECF6", - "gridcolor": "white", - "gridwidth": 2, - "linecolor": "white", - "showbackground": true, - "ticks": "", - "zerolinecolor": "white" - } - }, - "shapedefaults": { - "line": { - "color": "#2a3f5f" - } - }, - "ternary": { - "aaxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "baxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - }, - "bgcolor": "#E5ECF6", - "caxis": { - "gridcolor": "white", - "linecolor": "white", - "ticks": "" - } - }, - "title": { - "x": 0.05 - }, - "xaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - }, - "yaxis": { - "automargin": true, - "gridcolor": "white", - "linecolor": "white", - "ticks": "", - "title": { - "standoff": 15 - }, - "zerolinecolor": "white", - "zerolinewidth": 2 - } - } - }, - "title": { - "text": "Attention Visualization" - }, - "xaxis": { - "nticks": 378, - "tickangle": -90, - "title": { - "text": "Token in Input Sequence" - } - }, - "yaxis": { - "nticks": 1, - "title": { - "text": "Token in Input Sequence" - } - } - } - }, - "text/html": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "def visualize_attention(\n", " attention_weights,\n", @@ -3160,9 +222,11 @@ "\n", " # Generate token labels, marking special tokens with a special symbol\n", " x_token_labels = [\n", - " f\"{tokenizer.id_to_token(token)}\"\n", - " if tokenizer.id_to_token(token) in special_tokens\n", - " else str(i)\n", + " (\n", + " f\"{tokenizer.id_to_token(token)}\"\n", + " if tokenizer.id_to_token(token) in special_tokens\n", + " else str(i)\n", + " )\n", " for i, token in enumerate(concept_ids)\n", " ]\n", " y_token_labels = [\"[CLS]\"]\n", @@ -3370,13 +434,7 @@ "cell_type": "code", "execution_count": null, "id": "2e4ee069c3f083f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-15T13:59:33.204805Z", - "start_time": "2024-03-15T13:59:33.193223Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Visualize REG token -> Tricky?\n", diff --git a/odyssey/interp/grad_attribution.ipynb b/odyssey/interp/grad_attribution.ipynb index e09ec96..d44d55d 100644 --- a/odyssey/interp/grad_attribution.ipynb +++ b/odyssey/interp/grad_attribution.ipynb @@ -13,26 +13,19 @@ "metadata": {}, "outputs": [], "source": [ - "import argparse\n", "import os\n", - "import sys\n", - "from typing import Any, Dict, Optional\n", "\n", "import torch\n", - "from torch.utils.data import DataLoader\n", "\n", - "from odyssey.data.dataset import FinetuneDataset\n", "from odyssey.data.tokenizer import ConceptTokenizer\n", "from odyssey.interp.attribution import Attribution\n", - "from odyssey.models.cehr_bert.model import BertFinetune, BertPretrain\n", - "from odyssey.models.cehr_big_bird.model import BigBirdFinetune, BigBirdPretrain\n", + "from odyssey.interp.utils import get_type_id_mapping\n", "from odyssey.models.model_utils import (\n", " load_config,\n", " load_finetune_data,\n", " load_finetuned_model,\n", ")\n", - "from odyssey.utils.utils import seed_everything\n", - "from odyssey.interp.utils import get_type_id_mapping" + "from odyssey.utils.utils import seed_everything" ] }, { diff --git a/odyssey/models/baseline/Bi-LSTM.ipynb b/odyssey/models/baseline/Bi-LSTM.ipynb index 99dea78..7122853 100644 --- a/odyssey/models/baseline/Bi-LSTM.ipynb +++ b/odyssey/models/baseline/Bi-LSTM.ipynb @@ -2,13 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:49:56.508433100Z", - "start_time": "2024-02-09T04:49:56.494432800Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", @@ -34,53 +29,23 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:09.043998Z", - "start_time": "2024-02-09T04:49:56.503433500Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-02-08 23:50:03.831723: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", - "2024-02-08 23:50:04.033881: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-02-08 23:50:04.033946: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-02-08 23:50:04.046401: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-02-08 23:50:04.096988: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", - "2024-02-08 23:50:04.098640: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-02-08 23:50:06.342446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "import os\n", "import sys\n", "\n", + "\n", "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", "os.chdir(ROOT)\n", "\n", "from typing import Any, Dict, Tuple\n", - "from tqdm import tqdm\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", - "\n", "import torch\n", - "from torch import nn, optim\n", - "from torch.nn.functional import sigmoid\n", - "from torch.nn.utils.rnn import pack_padded_sequence\n", - "from torch.optim.lr_scheduler import ExponentialLR\n", - "from torch.utils.data import DataLoader, Dataset\n", - "\n", - "from odyssey.data.dataset import FinetuneDataset\n", - "from odyssey.models.cehr_big_bird.embeddings import Embeddings\n", - "from odyssey.data.tokenizer import ConceptTokenizer\n", "from sklearn.metrics import (\n", " auc,\n", " average_precision_score,\n", @@ -92,6 +57,17 @@ " roc_auc_score,\n", " roc_curve,\n", ")\n", + "from torch import nn, optim\n", + "from torch.nn.functional import sigmoid\n", + "from torch.nn.utils.rnn import pack_padded_sequence\n", + "from torch.optim.lr_scheduler import ExponentialLR\n", + "from torch.utils.data import DataLoader, Dataset\n", + "from tqdm import tqdm\n", + "\n", + "from odyssey.data.dataset import FinetuneDataset\n", + "from odyssey.data.tokenizer import ConceptTokenizer\n", + "from odyssey.models.cehr_big_bird.embeddings import Embeddings\n", + "\n", "\n", "DATA_ROOT = f\"{ROOT}/data/slurm_data/512/one_month\"\n", "DATA_PATH = f\"{DATA_ROOT}/pretrain.parquet\"\n", @@ -103,24 +79,13 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:09.149160700Z", - "start_time": "2024-02-09T04:50:09.043998Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cuda: NVIDIA A40\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# save parameters and configurations\n", + "\n", + "\n", "class config:\n", " \"\"\"A simple class to store all configurations\"\"\"\n", "\n", @@ -153,383 +118,9 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.198821600Z", - "start_time": "2024-02-09T04:50:09.150156900Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
patient_idnum_visitsdeceaseddeath_after_startdeath_after_endlengthtoken_lengthevent_tokens_512type_tokens_512age_tokens_512time_tokens_512visit_tokens_512position_tokens_512label
index
0f8f3289a-057f-5fcc-a714-5f6109ca16c420NaNNaN14[[CLS], [VS], 8938, [VE], [PAD], [PAD], [PAD],...[1, 2, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 8262, 8262, 8262, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 2, 2, 2, 513, 513, 513, 513, 513, 513, 513...0
19b62c9f4-3fdc-5020-82b5-ae5b8292445a40NaNNaN4352[[CLS], [VS], 7569, 66689036430, 00904224461, ...[1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 4, ...[0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28...[0, 5963, 5963, 5963, 5963, 5963, 5963, 5963, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
22ca522eb-dd89-5f79-8155-9599ea46b0b221244.0242.05154[[CLS], [VS], 00904629261, 00904642281, 009046...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86...[0, 8016, 8016, 8016, 8016, 8016, 8016, 8016, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
402adf8a6-8bc0-55d3-81ae-4d85820948969120.011.0640664[[CLS], [VS], 52007_3, 51476_2, 50861_1, 50862...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73...[0, 8426, 8426, 8426, 8426, 8426, 8426, 8426, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[1, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ...1
5744fe3c4-9b03-55ae-ac9f-6bc4e967cde730NaNNaN8086[[CLS], [VS], 7813, 7813, 7902, 7902, 9604, 00...[1, 2, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29...[0, 7582, 7582, 7582, 7582, 7582, 7582, 7582, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
.............................................
598983dafdc7c-c80b-56f0-832a-3ab7bf5667cc13111.010.019561992[[CLS], [VS], 51379_3, 51376_1, 51375_3, 51516...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81...[0, 9072, 9072, 9072, 9072, 9072, 9072, 9072, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[1, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11...1
16456317cd52f2-9a32-5b2d-aec8-bbaefce55e7b4116.05.018771886[[CLS], [VS], 50863_1, 50868_3, 50878_4, 50882...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58...[0, 6777, 6777, 6777, 6777, 6777, 6777, 6777, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, ...1
22291052ca40b-d12e-5390-a9f1-70a6edbfa1622163.024.018791882[[CLS], [VS], 50902_0, 50912_4, 50931_4, 50960...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89...[0, 6083, 6083, 6083, 6083, 6083, 6083, 6083, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...1
28930472730fd-189f-524c-a649-1f9d184e80c7612.00.010901105[[CLS], [VS], 51250_4, 51248_2, 51244_2, 51222...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81...[0, 6604, 6604, 6604, 6604, 6604, 6604, 6604, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...1
645158b689ccf-1b9e-578c-be35-c4c9887d886c30NaNNaN5359[[CLS], [VS], 9623, 58914050156, 00338011704, ...[1, 2, 7, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, ...[0, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41...[0, 8593, 8593, 8593, 8593, 8593, 8593, 8593, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
\n", - "

170593 rows × 14 columns

\n", - "
" - ], - "text/plain": [ - " patient_id num_visits deceased \\\n", - "index \n", - "0 f8f3289a-057f-5fcc-a714-5f6109ca16c4 2 0 \n", - "1 9b62c9f4-3fdc-5020-82b5-ae5b8292445a 4 0 \n", - "2 2ca522eb-dd89-5f79-8155-9599ea46b0b2 2 1 \n", - "4 02adf8a6-8bc0-55d3-81ae-4d8582094896 9 1 \n", - "5 744fe3c4-9b03-55ae-ac9f-6bc4e967cde7 3 0 \n", - "... ... ... ... \n", - "59898 3dafdc7c-c80b-56f0-832a-3ab7bf5667cc 13 1 \n", - "164563 17cd52f2-9a32-5b2d-aec8-bbaefce55e7b 4 1 \n", - "22291 052ca40b-d12e-5390-a9f1-70a6edbfa162 2 1 \n", - "28930 472730fd-189f-524c-a649-1f9d184e80c7 6 1 \n", - "64515 8b689ccf-1b9e-578c-be35-c4c9887d886c 3 0 \n", - "\n", - " death_after_start death_after_end length token_length \\\n", - "index \n", - "0 NaN NaN 1 4 \n", - "1 NaN NaN 43 52 \n", - "2 244.0 242.0 51 54 \n", - "4 20.0 11.0 640 664 \n", - "5 NaN NaN 80 86 \n", - "... ... ... ... ... \n", - "59898 11.0 10.0 1956 1992 \n", - "164563 16.0 5.0 1877 1886 \n", - "22291 63.0 24.0 1879 1882 \n", - "28930 2.0 0.0 1090 1105 \n", - "64515 NaN NaN 53 59 \n", - "\n", - " event_tokens_512 \\\n", - "index \n", - "0 [[CLS], [VS], 8938, [VE], [PAD], [PAD], [PAD],... \n", - "1 [[CLS], [VS], 7569, 66689036430, 00904224461, ... \n", - "2 [[CLS], [VS], 00904629261, 00904642281, 009046... \n", - "4 [[CLS], [VS], 52007_3, 51476_2, 50861_1, 50862... \n", - "5 [[CLS], [VS], 7813, 7813, 7902, 7902, 9604, 00... \n", - "... ... \n", - "59898 [[CLS], [VS], 51379_3, 51376_1, 51375_3, 51516... \n", - "164563 [[CLS], [VS], 50863_1, 50868_3, 50878_4, 50882... \n", - "22291 [[CLS], [VS], 50902_0, 50912_4, 50931_4, 50960... \n", - "28930 [[CLS], [VS], 51250_4, 51248_2, 51244_2, 51222... \n", - "64515 [[CLS], [VS], 9623, 58914050156, 00338011704, ... \n", - "\n", - " type_tokens_512 \\\n", - "index \n", - "0 [1, 2, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 4, ... \n", - "2 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "5 [1, 2, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "... ... \n", - "59898 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "164563 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "22291 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "28930 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "64515 [1, 2, 7, 6, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, ... \n", - "\n", - " age_tokens_512 \\\n", - "index \n", - "0 [0, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28... \n", - "2 [0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86... \n", - "4 [0, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73... \n", - "5 [0, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29... \n", - "... ... \n", - "59898 [0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81... \n", - "164563 [0, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58, 58... \n", - "22291 [0, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89, 89... \n", - "28930 [0, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, 81... \n", - "64515 [0, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41... \n", - "\n", - " time_tokens_512 \\\n", - "index \n", - "0 [0, 8262, 8262, 8262, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 5963, 5963, 5963, 5963, 5963, 5963, 5963, ... \n", - "2 [0, 8016, 8016, 8016, 8016, 8016, 8016, 8016, ... \n", - "4 [0, 8426, 8426, 8426, 8426, 8426, 8426, 8426, ... \n", - "5 [0, 7582, 7582, 7582, 7582, 7582, 7582, 7582, ... \n", - "... ... \n", - "59898 [0, 9072, 9072, 9072, 9072, 9072, 9072, 9072, ... \n", - "164563 [0, 6777, 6777, 6777, 6777, 6777, 6777, 6777, ... \n", - "22291 [0, 6083, 6083, 6083, 6083, 6083, 6083, 6083, ... \n", - "28930 [0, 6604, 6604, 6604, 6604, 6604, 6604, 6604, ... \n", - "64515 [0, 8593, 8593, 8593, 8593, 8593, 8593, 8593, ... \n", - "\n", - " visit_tokens_512 \\\n", - "index \n", - "0 [0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, ... \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "5 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "... ... \n", - "59898 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "164563 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "22291 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "28930 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "64515 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "\n", - " position_tokens_512 label \n", - "index \n", - "0 [0, 2, 2, 2, 513, 513, 513, 513, 513, 513, 513... 0 \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "4 [1, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, ... 1 \n", - "5 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "... ... ... \n", - "59898 [1, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11... 1 \n", - "164563 [1, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, ... 1 \n", - "22291 [1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ... 1 \n", - "28930 [1, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... 1 \n", - "64515 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "\n", - "[170593 rows x 14 columns]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Load data\n", "pretrain_data = pd.read_parquet(DATA_PATH)\n", @@ -555,13 +146,8 @@ }, { "cell_type": "code", - "execution_count": 6, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.209725Z", - "start_time": "2024-02-09T04:50:22.200726900Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Define custom labels if dataset does not already have labels\n", @@ -572,25 +158,9 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.369872300Z", - "start_time": "2024-02-09T04:50:22.212728Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Fit tokenizer on .json vocab files\n", "tokenizer = ConceptTokenizer(data_dir=config.data_dir)\n", @@ -602,25 +172,13 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.494688Z", - "start_time": "2024-02-09T04:50:22.424633300Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Data is ready to go!\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Define dataset with token lengths\n", + "\n", + "\n", "class DatasetWithTokenLength(Dataset):\n", " def __init__(self, tokenized_data: FinetuneDataset, length_data: np.ndarray):\n", " super(Dataset, self).__init__()\n", @@ -696,16 +254,13 @@ }, { "cell_type": "code", - "execution_count": 9, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.503675700Z", - "start_time": "2024-02-09T04:50:22.494688Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Define model architecture\n", + "\n", + "\n", "class BiLSTMModel(nn.Module):\n", " def __init__(\n", " self,\n", @@ -780,13 +335,8 @@ }, { "cell_type": "code", - "execution_count": 10, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T04:50:22.515675900Z", - "start_time": "2024-02-09T04:50:22.514675600Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Set hyperparameters for Bi-LSTM model adn training loop\n", @@ -803,46 +353,9 @@ }, { "cell_type": "code", - "execution_count": 11, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T05:58:12.850427800Z", - "start_time": "2024-02-09T04:50:22.518700100Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Adjusting learning rate of group 0 to 1.0000e-03.\n", - "Epoch 1/6: 100%|██████████| 2666/2666 [11:34<00:00, 3.84 batch/s]\n", - "\n", - "Epoch 1/6 | Average Train Loss: 1.04758 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 7.5000e-04.\n", - "Epoch 2/6: 100%|██████████| 2666/2666 [11:32<00:00, 3.85 batch/s]\n", - "\n", - "Epoch 2/6 | Average Train Loss: 0.79291 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 5.6250e-04.\n", - "Epoch 3/6: 100%|██████████| 2666/2666 [11:21<00:00, 3.91 batch/s]\n", - "\n", - "Epoch 3/6 | Average Train Loss: 0.72944 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 4.2188e-04.\n", - "Epoch 4/6: 100%|██████████| 2666/2666 [11:10<00:00, 3.98 batch/s]\n", - "\n", - "Epoch 4/6 | Average Train Loss: 0.70437 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 3.1641e-04.\n", - "Epoch 5/6: 100%|██████████| 2666/2666 [11:06<00:00, 4.00 batch/s]\n", - "\n", - "Epoch 5/6 | Average Train Loss: 0.68042 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 2.3730e-04.\n", - "Epoch 6/6: 100%|██████████| 2666/2666 [10:55<00:00, 4.07 batch/s]\n", - "\n", - "Epoch 6/6 | Average Train Loss: 0.66466 | Train Accuracy: 0.00000 | \n", - "Adjusting learning rate of group 0 to 1.7798e-04.\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Training Loop\n", "model = BiLSTMModel(input_size, hidden_size, num_layers, output_size, dropout_rate).to(\n", @@ -914,9 +427,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Save the model if needed\n", @@ -943,22 +454,9 @@ }, { "cell_type": "code", - "execution_count": 12, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T05:58:20.707426Z", - "start_time": "2024-02-09T05:58:12.855429600Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test Evaluation: 100%|██████████| 47/47 [00:07<00:00, 6.45 batch/s]\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Assess model performance and get predictions for train and test set\n", "y_train_pred = np.array([])\n", @@ -1014,14 +512,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-09T05:58:20.715428Z", - "start_time": "2024-02-09T05:58:20.707426Z" - }, - "collapsed": false - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Save model predictions, labels, and probabilities to disk\n", diff --git a/odyssey/models/baseline/XGBoost.ipynb b/odyssey/models/baseline/XGBoost.ipynb index 37068ee..f567455 100644 --- a/odyssey/models/baseline/XGBoost.ipynb +++ b/odyssey/models/baseline/XGBoost.ipynb @@ -3,12 +3,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:17:53.451568Z", - "start_time": "2024-03-18T02:17:53.443199Z" - } - }, + "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", @@ -33,13 +28,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:17:54.970742Z", - "start_time": "2024-03-18T02:17:53.455747Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Import dependencies and define useful constants\n", @@ -93,468 +83,9 @@ }, { "cell_type": "code", - "execution_count": 32, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:18:02.554807Z", - "start_time": "2024-03-18T02:17:54.972637Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
patient_idnum_visitsdeceaseddeath_after_startdeath_after_endlengthtoken_lengthevent_tokens_2048type_tokens_2048age_tokens_2048time_tokens_2048visit_tokens_2048position_tokens_2048elapsed_tokens_2048common_conditionsrare_conditionslast_VS_indexlast_VE_indexlength_of_staylabel
035581927-9c95-5ae9-af76-7d74870a349c10NaNNaN5040[CLS] [VS] 00006473900 00904516561 51079000220...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ...[0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85...[0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1522.0416670
1f5bba8dd-25c0-5336-8d3d-37424c18502620NaNNaN14881[CLS] [VS] 52135_2 52075_2 52074_2 52073_3 520...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83...[0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[0, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]681543.5000000
2f4938f91-cadb-5133-8541-a52fb0916cea20NaNNaN7886[CLS] [VS] 0RB30ZZ 0RG10A0 00071101441 0090419...[1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44...[0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]49841.3750000
36fe2371b-a6f0-5436-aade-7795005b0c6620NaNNaN8691[CLS] [VS] 63739057310 49281041688 00597026010...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72...[0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7...[1, 0, 0, 0, 0, 0, 0, 1, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]55921.4583330
46f7590ae-f3b9-50e5-9e41-d4bb1000887a10NaNNaN7256[CLS] [VS] 50813_0 52135_0 52075_3 52074_3 520...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47...[0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[1, 0, 0, 0, 0, 0, 0, 0, 0, 1][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1742.9583330
...............................................................
1434743f300d4e-4554-5f1f-9dff-f209a4916cbc70NaNNaN536564[CLS] [VS] 51484_0 51146_3 51200_1 51221_4 512...[1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64...[0, 6921, 6921, 6921, 6921, 6921, 6921, 6921, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0...[1, 1, 1, 1, 0, 0, 0, 0, 1, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]5205621.6666670
143475cf2115d7-937e-511d-b159-dd7eb3d5d42020NaNNaN166142[CLS] [VS] 33332001001 00781305714 10019017644...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 6, 5, ...[0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32...[0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 1.16, 1.25, 1.3, 1.3, 1.31, 1.53,...[1, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1091722.5416670
14347631338a39-28f9-54a5-a810-2d05fbaa516630NaNNaN283221[CLS] [VS] 00338011704 00409128331 63323026201...[1, 2, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, ...[0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50...[0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 1.48, 1.49, 1.52, 1.52, 1.6, 1.6,...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1922934.1250000
1434770989415d-394c-5f42-8dac-75dc7306a23c20NaNNaN470140[CLS] [VS] 49281041550 51079043620 51079088120...[1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 4, 2, 7, 6, ...[0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0,...[0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, ...[-2.0, -1.0, 1.3, 1.78, 1.78, 1.78, 1.84, 1.94...[0, 0, 0, 0, 1, 0, 0, 0, 1, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]1247619.2500001
14347802cee673-6875-50c9-bad9-e7a7746731eb20NaNNaN98106[CLS] [VS] 33332001101 00904224461 00904516561...[1, 2, 6, 6, 6, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52...[0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...[-2.0, -1.0, 2.64, 2.71, 2.71, 4.15, 4.15, 4.4...[1, 0, 1, 1, 0, 1, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0]391042.6250000
\n", - "

143479 rows × 20 columns

\n", - "
" - ], - "text/plain": [ - " patient_id num_visits deceased \\\n", - "0 35581927-9c95-5ae9-af76-7d74870a349c 1 0 \n", - "1 f5bba8dd-25c0-5336-8d3d-37424c185026 2 0 \n", - "2 f4938f91-cadb-5133-8541-a52fb0916cea 2 0 \n", - "3 6fe2371b-a6f0-5436-aade-7795005b0c66 2 0 \n", - "4 6f7590ae-f3b9-50e5-9e41-d4bb1000887a 1 0 \n", - "... ... ... ... \n", - "143474 3f300d4e-4554-5f1f-9dff-f209a4916cbc 7 0 \n", - "143475 cf2115d7-937e-511d-b159-dd7eb3d5d420 2 0 \n", - "143476 31338a39-28f9-54a5-a810-2d05fbaa5166 3 0 \n", - "143477 0989415d-394c-5f42-8dac-75dc7306a23c 2 0 \n", - "143478 02cee673-6875-50c9-bad9-e7a7746731eb 2 0 \n", - "\n", - " death_after_start death_after_end length token_length \\\n", - "0 NaN NaN 50 40 \n", - "1 NaN NaN 148 81 \n", - "2 NaN NaN 78 86 \n", - "3 NaN NaN 86 91 \n", - "4 NaN NaN 72 56 \n", - "... ... ... ... ... \n", - "143474 NaN NaN 536 564 \n", - "143475 NaN NaN 166 142 \n", - "143476 NaN NaN 283 221 \n", - "143477 NaN NaN 470 140 \n", - "143478 NaN NaN 98 106 \n", - "\n", - " event_tokens_2048 \\\n", - "0 [CLS] [VS] 00006473900 00904516561 51079000220... \n", - "1 [CLS] [VS] 52135_2 52075_2 52074_2 52073_3 520... \n", - "2 [CLS] [VS] 0RB30ZZ 0RG10A0 00071101441 0090419... \n", - "3 [CLS] [VS] 63739057310 49281041688 00597026010... \n", - "4 [CLS] [VS] 50813_0 52135_0 52075_3 52074_3 520... \n", - "... ... \n", - "143474 [CLS] [VS] 51484_0 51146_3 51200_1 51221_4 512... \n", - "143475 [CLS] [VS] 33332001001 00781305714 10019017644... \n", - "143476 [CLS] [VS] 00338011704 00409128331 63323026201... \n", - "143477 [CLS] [VS] 49281041550 51079043620 51079088120... \n", - "143478 [CLS] [VS] 33332001101 00904224461 00904516561... \n", - "\n", - " type_tokens_2048 \\\n", - "0 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, ... \n", - "1 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "2 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "3 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "4 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "... ... \n", - "143474 [1, 2, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "143475 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, 6, 5, ... \n", - "143476 [1, 2, 6, 6, 6, 6, 5, 5, 5, 5, 5, 5, 5, 5, 5, ... \n", - "143477 [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 8, 4, 2, 7, 6, ... \n", - "143478 [1, 2, 6, 6, 6, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "\n", - " age_tokens_2048 \\\n", - "0 [0, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, 85... \n", - "1 [0, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83, 83... \n", - "2 [0, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 44... \n", - "3 [0, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72, 72... \n", - "4 [0, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47... \n", - "... ... \n", - "143474 [0, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64... \n", - "143475 [0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32... \n", - "143476 [0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50... \n", - "143477 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0,... \n", - "143478 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52... \n", - "\n", - " time_tokens_2048 \\\n", - "0 [0, 5902, 5902, 5902, 5902, 5902, 5902, 5902, ... \n", - "1 [0, 6594, 6594, 6594, 6594, 6594, 6594, 6594, ... \n", - "2 [0, 8150, 8150, 8150, 8150, 8150, 8150, 8150, ... \n", - "3 [0, 6093, 6093, 6093, 6093, 6093, 6093, 6093, ... \n", - "4 [0, 6379, 6379, 6379, 6379, 6379, 6379, 6379, ... \n", - "... ... \n", - "143474 [0, 6921, 6921, 6921, 6921, 6921, 6921, 6921, ... \n", - "143475 [0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ... \n", - "143476 [0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ... \n", - "143477 [0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ... \n", - "143478 [0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ... \n", - "\n", - " visit_tokens_2048 \\\n", - "0 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "3 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "... ... \n", - "143474 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "143475 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "143476 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "143477 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, ... \n", - "143478 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "\n", - " position_tokens_2048 \\\n", - "0 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "1 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "2 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "3 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "4 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "... ... \n", - "143474 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "143475 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "143476 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "143477 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, ... \n", - "143478 [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... \n", - "\n", - " elapsed_tokens_2048 \\\n", - "0 [-2.0, -1.0, 1.97, 2.02, 2.02, 2.02, 2.02, 2.0... \n", - "1 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "2 [-2.0, -1.0, 0.0, 0.0, 1.08, 1.08, 13.89, 13.8... \n", - "3 [-2.0, -1.0, 0.75, 0.75, 0.75, 0.75, 0.75, 0.7... \n", - "4 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "... ... \n", - "143474 [-2.0, -1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0... \n", - "143475 [-2.0, -1.0, 1.16, 1.25, 1.3, 1.3, 1.31, 1.53,... \n", - "143476 [-2.0, -1.0, 1.48, 1.49, 1.52, 1.52, 1.6, 1.6,... \n", - "143477 [-2.0, -1.0, 1.3, 1.78, 1.78, 1.78, 1.84, 1.94... \n", - "143478 [-2.0, -1.0, 2.64, 2.71, 2.71, 4.15, 4.15, 4.4... \n", - "\n", - " common_conditions rare_conditions \\\n", - "0 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "1 [0, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "3 [1, 0, 0, 0, 0, 0, 0, 1, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "4 [1, 0, 0, 0, 0, 0, 0, 0, 0, 1] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "... ... ... \n", - "143474 [1, 1, 1, 1, 0, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "143475 [1, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "143476 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "143477 [0, 0, 0, 0, 1, 0, 0, 0, 1, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "143478 [1, 0, 1, 1, 0, 1, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", - "\n", - " last_VS_index last_VE_index length_of_stay label \n", - "0 1 52 2.041667 0 \n", - "1 68 154 3.500000 0 \n", - "2 49 84 1.375000 0 \n", - "3 55 92 1.458333 0 \n", - "4 1 74 2.958333 0 \n", - "... ... ... ... ... \n", - "143474 520 562 1.666667 0 \n", - "143475 109 172 2.541667 0 \n", - "143476 192 293 4.125000 0 \n", - "143477 12 476 19.250000 1 \n", - "143478 39 104 2.625000 0 \n", - "\n", - "[143479 rows x 20 columns]" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Load dataset\n", "dataset = pd.read_parquet(\n", @@ -569,14 +100,8 @@ }, { "cell_type": "code", - "execution_count": 33, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:18:02.652563Z", - "start_time": "2024-03-18T02:18:02.557851Z" - }, - "collapsed": false - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "patient_ids = pickle.load(open(ID_PATH, \"rb\"))\n", @@ -588,38 +113,9 @@ }, { "cell_type": "code", - "execution_count": 22, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:18:07.521714Z", - "start_time": "2024-03-18T02:18:02.654221Z" - } - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Loading Tokens: 100%|██████████| 143479/143479 [00:06<00:00, 20885.94 Patients/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Complete list of unique event tokens\n", - "Length: 18534\n", - "Head: ['[W_3]', '[W_2]', '[W_1]', '[W_0]', '[VS]', '[VE]', '[REG]', '[M_9]', '[M_8]', '[M_7]', '[M_6]', '[M_5]', '[M_4]', '[M_3]', '[M_2]', '[M_1]', '[M_12]', '[M_11]', '[M_10]', '[M_0]', '[LT]', '[CLS]', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033H4', 'XW033B3']...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Find the unique set of all possible tokens, including special tokens\n", "unique_event_tokens = set()\n", @@ -642,22 +138,9 @@ }, { "cell_type": "code", - "execution_count": 23, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:18:07.534354Z", - "start_time": "2024-03-18T02:18:07.523148Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "18513 ['id', 'XY0VX83', 'XW0DXR5', 'XW0DX82', 'XW043C3', 'XW043B3', 'XW04351', 'XW033H4', 'XW033B3', 'XW03372', 'XW03331', 'X2RF332', 'X2RF032', 'X2C1361', 'X2C0361', 'X2A5312', 'HZ99ZZZ', 'HZ87ZZZ', 'HZ85ZZZ', 'HZ81ZZZ']\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Define the list of tokens being used as features for the XGBoost\n", "# Note that feature 'id' will be dropped later on\n", @@ -685,17 +168,14 @@ }, { "cell_type": "code", - "execution_count": 24, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:25:27.580887Z", - "start_time": "2024-03-18T02:22:17.577626Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Generate feature frequency matrix for each feature token in the patient data\n", "# Since this will be saved to disk, you need only do it once\n", + "\n", + "\n", "def get_patient_frequency_matrix(\n", " data: pd.DataFrame,\n", " feature_event_tokens: List[str],\n", @@ -762,26 +242,9 @@ }, { "cell_type": "code", - "execution_count": 25, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:37.007429Z", - "start_time": "2024-03-18T02:41:36.683489Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "<50000x18513 sparse matrix of type ''\n", - "\twith 5122992 stored elements in Compressed Sparse Row format>" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Load feature frequency matrix from disk, generated by above code cell\n", "patient_freq_matrix = load_npz(FREQ_MATRIX_TRAIN)\n", @@ -792,13 +255,8 @@ }, { "cell_type": "code", - "execution_count": 26, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:37.481671Z", - "start_time": "2024-03-18T02:41:37.472431Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "def find_min_greater_than_zero(lst: List[int]) -> int:\n", @@ -814,17 +272,13 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:40.198998Z", - "start_time": "2024-03-18T02:41:38.324974Z" - }, - "collapsed": false - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Get extra features such as number of visits and age\n", + "\n", + "\n", "def add_age_to_freq_matrix(\n", " data: pd.DataFrame,\n", " patient_freq_matrix: np.ndarray,\n", @@ -853,23 +307,9 @@ }, { "cell_type": "code", - "execution_count": 28, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:40.227252Z", - "start_time": "2024-03-18T02:41:40.200678Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "How many features have been reported for at least 1 patients?\n", - "14443 Features\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Get intuition about the frequency of different features in the training dataset\n", "report_threshold = 1\n", @@ -892,25 +332,9 @@ }, { "cell_type": "code", - "execution_count": 29, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:40.255936Z", - "start_time": "2024-03-18T02:41:40.228722Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "array([18514, 18512, 18513, ..., 7736, 9232, 14290])" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Pick features to train the model on\n", "NUM_FEATURES = 20000\n", @@ -921,14 +345,8 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:40.261325Z", - "start_time": "2024-03-18T02:41:40.258323Z" - }, - "collapsed": false - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Define custom labels, here death in 12 M, not needed if dataset already has labels\n", @@ -941,13 +359,8 @@ }, { "cell_type": "code", - "execution_count": 50, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:42.023770Z", - "start_time": "2024-03-18T02:41:41.994187Z" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Prepare data for model training\n", @@ -966,14 +379,8 @@ }, { "cell_type": "code", - "execution_count": 38, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:42.557970Z", - "start_time": "2024-03-18T02:41:42.551Z" - }, - "collapsed": false - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# Split data into train and test based on original dataset definitions\n", @@ -1211,462 +618,9 @@ }, { "cell_type": "code", - "execution_count": 52, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:47.279093Z", - "start_time": "2024-03-18T02:41:42.845780Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
-       "              colsample_bylevel=None, colsample_bynode=None,\n",
-       "              colsample_bytree=None, device=None, early_stopping_rounds=None,\n",
-       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
-       "              gamma=None, grow_policy=None, importance_type=None,\n",
-       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
-       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
-       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
-       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
-       "              multi_strategy=None, n_estimators=None, n_jobs=None,\n",
-       "              num_parallel_tree=None, random_state=23, ...)
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" - ], - "text/plain": [ - "XGBClassifier(base_score=None, booster=None, callbacks=None,\n", - " colsample_bylevel=None, colsample_bynode=None,\n", - " colsample_bytree=None, device=None, early_stopping_rounds=None,\n", - " enable_categorical=False, eval_metric=None, feature_types=None,\n", - " gamma=None, grow_policy=None, importance_type=None,\n", - " interaction_constraints=None, learning_rate=None, max_bin=None,\n", - " max_cat_threshold=None, max_cat_to_onehot=None,\n", - " max_delta_step=None, max_depth=None, max_leaves=None,\n", - " min_child_weight=None, missing=nan, monotone_constraints=None,\n", - " multi_strategy=None, n_estimators=None, n_jobs=None,\n", - " num_parallel_tree=None, random_state=23, ...)" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Calculate class weights to scale positive weights\n", "total_negative = len(y_train) - sum(y_train)\n", @@ -1686,52 +640,9 @@ }, { "cell_type": "code", - "execution_count": 53, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:41:48.274852Z", - "start_time": "2024-03-18T02:41:47.281488Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Balanced Accuracy\n", - "Train: 0.86064 | Test: 0.77298 | All Data: 0.48225\n", - "\n", - "F1 Score\n", - "Train: 0.85604 | Test: 0.67183 | All Data: 0.58441\n", - "\n", - "Precision\n", - "Train: 0.88527 | Test: 0.60241 | All Data: 0.42988\n", - "\n", - "Recall\n", - "Train: 0.82868 | Test: 0.75934 | All Data: 0.91237\n", - "\n", - "AUROC\n", - "Train: 0.86064 | Test: 0.77298 | All Data: 0.48225\n", - "\n", - "AUC-PR\n", - "Train: 0.89980 | Test: 0.71681 | All Data: 0.69037\n", - "\n", - "Average Precision Score\n", - "Train: 0.81926 | Test: 0.52930 | All Data: 0.43071\n", - "\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0sAAAJuCAYAAABhUplHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzddXwU19oH8N+6ZnfjbiQkEMODBE0CBVqoO21p6Vtq1Evdvb116t7e3roLFiS4ewgWEuKezWZd5rx/LAwsERJIspHnez+97JyZnX02mezub+fMOQLGGAMhhBBCCCGEEA9CbxdACCGEEEIIIT0RhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQgghhBBCCGkBhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQgghhBBCCGkBhSVCCCGEEEIIaQGFJUIIIYQQQghpAYUlQshZefjhh5GYmNjsv5EjR2LevHnYvn2712pLTEzEf/7zH689PgDU1NTg5ZdfxvTp05GWloZhw4bhkksuwccffwyLxeLV2trr119/RWJiIgoKCrrtMRsbG/Hmm2/i/PPPx5AhQzBy5EhcdNFF+Oijj2A2m5vVVlpa2m219QSZmZm49957z/r+mzdvRmJiItasWXPOtdTV1WHy5Mn49NNPAZz9a8KyZcswb948jB49GikpKRg/fjzuuece7Nmzp9XH/vfffzF37lykp6cjNTUVmZmZeOSRR3Dw4EF+m7vuugtz586F0+ls93Pas2cP7rnnHowfPx4pKSmYMGECbrnlFqxatard+yCE9DGMEELOwkMPPcTGjBnDqqur+f8qKyvZ1q1b2bx581hycjLbvXu3V2qrrq5mRqPRK4/NGGN5eXlszJgx7LLLLmPLly9nxcXF7OjRo+z7779nEydOZLNmzWJ1dXVeq681b7/9NnvooYf4ZYvFwqqrq5nT6eyWxz927BibMmUKmzFjBvv7779ZcXExy8/PZ1988QUbNWoUmzVrFmtoaGCMMfbLL7+whIQEVlJS0i21nYvJkyezTZs2dcq+6urqmMFgaPf2P//8M5szZw6/bLPZWHV1NbPZbOdUh9PpZFdffTW79dZb+bazeU14/PHHWWpqKnvjjTfYvn37WFlZGdu6dSt78MEH2aBBg9i3337rsT3HcezBBx9kaWlp7J133mH5+fmsuLiY5eTksCuuuIKlpqaylStXMsYYa2pqYtnZ2ezFF19s13P6/vvv2aBBg9jChQvZ1q1bWWlpKduyZQt74IEHWEJCAnvzzTfP6WdGCOmdKCwRQs7KQw89xMaNG9fiOrPZzEaOHMkeeOCBbq7K+2w2G8vKymJXXXVVix9IKyoq2OjRo9mdd97pheraNnfuXI+w1N0uu+wyNnXq1BbDwL59+1hSUhJ7++23GWO9JyxVVlayhISETgtLHfXYY495hKXO8vPPP7OkpCR27Ngxvq2jrwk//vgjS0hI4MPN6f7zn/+wpKQklpeXx7d9++23LCEhga1atarZ9jabjV155ZUsOzubORwOxhhj//zzDxs0aBA7dOhQm88nPz+fJSUlsZdffrnF9S+99BJLSkpiR48ebXM/hJC+h7rhEUI6nUKhQHR0NCorK/k2xhi+/PJLXHjhhRg6dCjGjRuHJ598EgaDweO+ubm5uOyyy5CWlobJkyfjueeeg9Fo5NfX1NRg4cKFyMzMRGpqKs4//3z8/PPPHvs40Q2vuLgYiYmJ+OGHH5rVOGPGDNx2223tru3hhx/GhRdeiO+++w7p6el45ZVXWnzuy5YtQ0lJCR5++GFIpdJm60NCQnDLLbdg+fLlKCkpAQC8++67SElJwYEDB3DFFVcgLS0NEyZMwMcff+xx3/Y+948//hjz589Hamoq3y1pzZo1uPrqqzF06FAMGzYMF198MZYtW8bfLzMzExs2bMBvv/2GxMREbN68uVk3vBM/g82bN+OSSy7BkCFDMHXqVPz2228eNeTk5GDGjBlITU3FBRdcgNzcXMybNw/XXXddiz8zANi2bRv27NmDBQsWwMfHp9n65ORkLFmyBHfddZdHe0NDA+68804MGzYMI0aMwLPPPgu73c6vLywsxIIFC5Ceno6UlBRMnToVH3zwATiO47e57rrrcPvtt+Ott97CsGHD8N///rfd9wWA3377DbNmzUJaWhqys7Px9ttvw+l0YvPmzZg4cSIA4Prrr0dmZiZ/nz/++AOXX345hg8fjvT0dNx7772oqqri17/77rsYOXIkcnJyMH78eP55n94N74cffsCsWbMwdOhQjBo1CjfddBPy8vL45/XTTz9hy5YtSExMxK+//tpiN7zdu3fjuuuuw9ChQzF+/HgsXLgQNTU1rf6uGGN47733cMEFFyAqKqrV7U7V0mvCp59+igkTJmDKlCkt3mfBggVQq9X4+uuv+bbPP/8cGRkZmDx5crPtpVIp3n33Xfz5558Qi8UA3H/n0dHReO+999qs7+uvv4ZCoWh2fJ1w9913Izc3F7GxsQDcP9srrrjCY5vTf7Yt/Q6vueYaXHXVVc32//HHHyM5ORl1dXUA3L+TefPmYdy4cRg6dCiuvfZa7Nixo83nQAjpGhSWCCGdzm63o6SkBGFhYXzbBx98gJdffhnnn38+/vzzT7z88stYt24d7rzzTn6bbdu24dZbb0VGRgZ+++03vPTSS1i2bBkee+wxfr833HADtm/fjqeffhp//fUXLrzwQjz++OP4/fffm9URFRWFIUOGYOnSpR7tBw4cwNGjR3HhhRe2uzbA/cE8JycH33zzDebPn9/ic9+0aRN0Oh2GDBnS6s9n8uTJYIxh8+bNfJvD4cCzzz6Le++9F3/88Qdmz56N119/Hf/++2+Hn/tPP/2EESNGYPHixYiNjUVxcTFuv/12DBgwAL///jv++OMP/rqQ/fv3AwB+/vln+Pn5YcaMGVi3bh2GDRvWYu319fVYtGgR/7hxcXF44oknUFFRAQA4fPgw7r77bkRFReGnn37C448/jtdff/2M1z1t3rwZAoEAkyZNanWbyMjIZm0vv/wyLrjgAvz++++46aab8O233+KPP/4A4P5Qf8stt6CiogJffvklli5dirvvvhvvvfcevv32W4/9HDp0CMeOHcMvv/yCCy+8sN33/euvv/DYY4/h0ksvxV9//YWHH34YX375Jd544w0MGzYMr7/+OgD3B+cTwfaPP/7AwoULMXToUPz66694//33cfToUcydO9cj6LlcLnzzzTf44IMP8PTTTzd77hs3bsTTTz+NG2+8Ef/88w+++eYbaLVa3HTTTbBYLHj33XeRnJyMYcOGYd26dZg5c2azfRQVFWHu3LmIjIzEjz/+iEWLFmH//v38FwktycvLQ1lZmUf4O5PTXxOqqqpQVFTUYug5QSqVIiMjA5s2bQIAVFRUoKSkpM1jJDAwEAqFgl8WCASYMmUKcnNzPX62p9uyZQtGjx7tcd9TKRQKBAQEtPUUW3T673DWrFnYtWuXRzAG3NdgjR8/Hv7+/igsLMQNN9wAl8uFTz75BD/88ANCQkJw0003dev1g4QQNwpLhJBOVVtbiyeffBJGoxHXXHMNAHcQ+Oyzz3DhhRfilltuQVRUFCZOnIhHH30Umzdv5r8x/fTTT5GQkIB7770XcXFxGDt2LB5//HGoVCo4HA7k5OSgoKAAL7zwAiZOnIiYmBjccsstyMzMxAcffNBiPbNmzcLmzZvR0NDAt/3777/QaDTIzMxsd22A+wPeQw89hMTEROh0uhYfr7Ky0iMktiQiIoLf9lRz5szB2LFjERsbiwcffBBhYWH4888/AaBDz93Hxwe33HILIiIiIJVKERwcjD/++AOPPfYYYmJiEBUVhTvvvBMulwsbNmwAAPj5+UEoFEIulyMwMLDFs2IAUF1djSeeeALDhw9HbGws5s2bB4fDwYeuv//+GwDw6quvYtCgQRgzZgxee+01Pky1pqqqCj4+PtBoNG1ud7qZM2di+vTpiI6Oxm233QalUukxMMDnn3+ODz/8EElJSQgPD8cFF1yApKQkrF271mM/lZWVePrppzFgwAD+zFZ77vvxxx9j8uTJmDt3LqKjo5GdnY2FCxfC5XJBKpXyz0er1cLPzw8A8OGHH2LUqFH872PkyJF4+eWXcfToUY9gbzabMXfuXKSmpvL3PdW+ffugUCgwe/ZshIeHY9CgQXjhhRfw8ccfQyQSQafTQSwWQyKRIDAwEHK5vNk+vvnmG8hkMjz77LNISEjA0KFD+Z/DibMcp9u6dSsAYNSoUe36HbX0mnDi2A8NDW3zvuHh4aiqqgLHcXzACA8Pb9fjnjBy5EiYzWb+jFtLqqqqOrzf9jj9dzh9+nSIxWIsWbKE36awsBD5+fn8lzdffvklhEIhH3YTExPx4osvQqVS4csvv+z0GgkhbRN7uwBCSO9VV1fncQaC4zhYrVYkJyfj/fff58+uFBQUwGg0IiMjw+P+Y8aMAQDs378fw4cPx549e5Cdne2xzXnnnYfzzjsPgLtrikQiQXp6usc2Y8eOxYoVK2AymaBSqTzWzZw5Ey+99BJycnJw+eWXAwAWL16M6dOnQyqV4sCBA+2qDQBkMhkSEhLa/JkIBAK4XK42t2GM8dueasSIER7LgwcP5rvqdeS5p6SkeGwjk8lw5MgRPPvssygoKIDJZOLX6fX6Nms9nVKp9PgZnPgQf6LLYnFxMaKioqDVavltEhMTzxggBQIB/3PpiKFDh/K3hUIhdDod//wEAgEMBgPeeOMN7N69G3q9HowxWK1WpKameuwnIiLCo+b23NdqteLQoUO44IILPPZ19dVXt1qv0WjE0aNHMXv2bI/2wYMHQ6fTYf/+/Zg1axbffvrv8lQZGRl47733cOWVV+Kyyy7DmDFjEBsb2+ZZzdPt2bMHycnJfLc1wB0uRo4c2ep9qqurIRaL4evr22xde18TThz7p3dpPB1jDEKhEAKBoNnfS3sFBgYCQJtdC8/2+GuPU3+Hvr6+GD9+PJYuXYobbrgBgPvLG7VazZ+p27NnD4YMGeLRHVUmk2H48OFtBj5CSNegsEQIOWs6nc7jeqB9+/bhvvvuw7x58zy6ypy45ujxxx/HU0891Ww/Jz7EGAyGZmHnVEajEQ6Ho1moODE0cE1NTbP7+/v7Y+zYsViyZAkuv/xy7N27F8XFxXjppZc6VBuAFq+lOV1YWBh27NgBxlirH+5OBKDTv8k+/ayKUqlEU1MTX2d7n/vp+1m+fDnuuusuTJ8+HW+99RYCAgIgEAgwbdq0Mz6f0ymVyhbbT3zQ1Ov1Lf4OW/pgfaqwsDA0NTWhvr6+xbMorTn9bMmpH3orKiowZ84cREdH48knn0RkZCTEYjEeeOCBZvs5/WfWnvueCIhtHbOnO3G8vffee82uSbNYLKiurm6zrlMlJSXhhx9+wOeff4533nkHTz/9NOLj43HfffchKyurXfUYDIYznt05XVNTE9RqdYvHd3tfE06E5xN/C60pKSlBaGgoBAIBX2dRUVGH6j3xMzz9+shThYaG4tixYx3ab0cf/4RZs2bh/vvvR1VVFYKDg7F48WKcd955/LFsNBpx8ODBZl1h7XZ7h/42CCGdg8ISIeSsiUQiREdH88vR0dFYunQpXnjhBWRkZPBd1U58Y//ggw/yF7yf6kQI8ff3R2NjY6uPp9FoIJfLW7w+CWi9S8+sWbPw2GOPQa/X499//0V4eDgfOtpbW3uNGzcO33//PTZt2oSxY8e2uE1ubi5EIlGz9SaTyeOaCZPJxH/QOtvnDgB//vkngoOD8eabb0IodPe+Pv1DeWeRSqWwWq3N2lsLUSec+FksW7asxQvgAWDp0qWIj49HXFxcu2rJycmB2WzGG2+8gQEDBvDtBoPB4yzS2d7X19cXQqGwzWP2dCeOp7lz5/JnOk/VWhhtTWJiIl555RUwxrB371588sknWLBgAf7991/ExMSc8f5n+ptriY+PD4xGY4tfCLT3NSEgIAAJCQnIycnBTTfd1OLj2O12bNq0CdOnTwcABAUFIS4uDsuXL8fNN9/c4n2Ki4uxb98+zJgxg6/tREhqK3iOHTsWv/32GxobG1s8NhwOB3744Qdceuml/N/o6WeiTp0HrC2ZmZlQKBRYtmwZxowZg8OHD+OJJ57g12s0GoSEhOD5559vdt8Tf7+EkO5Df3WEkE716KOPwmKx8GduACA2NhYajQYlJSWIjo7m/4uIiIDT6eS/LU1ISMC2bds89rd8+XJce+21MJlMGDp0KKxWKywWi8d+5HI5NBpNq9fZTJ06FWKxGKtXr8bSpUsxe/Zs/oNUe2trr6ysLMTExODVV19tcfLZqqoqfPbZZ5g9ezaCg4M91m3ZssVjef/+/fwH9bN97oD7g55Wq/X4oHViBLvTP/Cda1ek6OhoFBUVeXwA37dvH8rKytq8X1paGkaNGoVFixY1u/gdcP8sFi5ciO+//77dtTgcDgDw+B3u2LEDRUVFZ3ye7bmvRCJBbGwsfw3PCf/73/9wyy23eLSduI9KpUJCQgIKCws9fo/R0dGw2+3w9/dv9/Pbvn07du/eDcB9Ri0tLQ3PP/88XC4XDh061OyxW5KQkIC9e/d6BNxdu3bh6quvRnFxcYv3CQoKgtPpRH19fbvqbOk1AQA/Ue2J69xO995778FsNuP666/3uM+uXbvwyy+/NNvebrfjsccewyuvvOLR1fTE2eET3fFact1118HlcjWr8YR33nkHL774Io4cOQLAHWhOf/67du1qdf+nUigUyM7ORk5ODpYsWYKwsDCP7rVDhw5FYWEhQkNDPY4PxhiCgoLa9RiEkM5DYYkQ0qlCQkKwYMEC/P7771i/fj0AQCwW4+abb8Z3332Hr7/+GkVFRcjPz8cjjzyCyy+/nP9wPG/ePJSUlOC5555DSUkJNm3ahJdeegl+fn5QqVSYMmUKEhIS8MADD2DDhg0oKytDbm4u5syZ4/HN7OlUKhUyMzPx1VdfoaysjL+QuiO1tZdYLMbbb7+NyspKXHnllViyZAlKSkpQWFiIn376CVdccQUiIiL4Ef5O9dVXX2HdunUoLCzEq6++isrKSlx88cUAcNbPHXB/+Dpy5Aj+/fdflJSU4LPPPsPu3bsRGhqK/fv382eZNBoN9u/fj/z8fNTW1nboeZ8wY8YMfmS/I0eOYMuWLXjqqafadfH8K6+8AplMhiuuuAI///wzjh07hiNHjuCrr77C3LlzMXz4cI9hs8/kxPVMH330EUpLS5GTk4Nnn30WU6ZM4X8nrV0z09773nLLLdi4cSM+/PBDlJWVYeXKlXjrrbf4kHviLMX69euxf/9+MMYwf/58rFixAu+++y4KCgpw5MgRvPLKK7j44ov5gTLaY9WqVbj99tuxbNkylJWV4ejRo/jwww8hl8v566o0Gg2Kioqwd+/eFgfZOBESFi5ciMLCQuzZs4cffr2l0QeBkwM7nB4SW9PSawIAXHTRRbjyyivx8MMP49VXX8W+fftQXl6Obdu24eGHH8ann36K559/3uNM4qWXXopLL70UTzzxBF588UXs27cPpaWlyM3NxXXXXYdDhw7hrbfeglqt5u+zdetWKJVKJCcnt1pjXFwcnnnmGfz555+YP38+Nm7ciLKyMuzcuRMLFy7Ep59+iscff5z/uaalpaG0tBQ//vgjSkpK8OuvvyI3N7ddPw/AfbZ7+/btWLJkCWbNmuVxhu7666+HyWTC/fffj71796KkpAQ//vgjLrroohanQSCEdLHumtCJENK3tDUBpcPhYBdccAHLzMxkZrOZb//mm2/Y9OnTWXJyMhs2bBi75ZZb2P79+z3uu2LFCnbxxRezlJQUNnHiRPbss8+ypqYmfn1tbS17+OGH2ZgxY9jgwYPZhAkT2Msvv+zxOAkJCey1115rtt+EhAR26aWXtljzmWpr6/m2pLa2lr388svsvPPOY6mpqWzo0KHskksuYV988UWzyWrfeecdlpCQwHbu3MmuvPJKlpKSwjIyMtinn37abJ9n89xNJhN78MEH2ciRI9moUaPYwoULWVNTE/v8889ZWloau+666xhjjP32229s5MiRbMiQIezff//lJ349cuRIqz+DI0eOsISEBPbLL7/wbT/++CObMmUKS01NZZdccgnbunUru+SSS9hNN910xp+bwWBgb775Jps5cyZLS0tjo0aNYpdffjn77rvvmN1u57c7vbYTpkyZwu655x5++aOPPmLjx49nQ4cOZddddx07cuQI27p1Kxs9ejRLT09nBoOBzZkzh11++eXNamnPfRlj7KeffmIzZsxgKSkpLDMzk7399tt8rS6Xi912220sOTmZjR07ljmdTsaYe7LUiy66iCUnJ7MhQ4awOXPmsI0bN/KPfeKYsFqtrT4/u93O3njjDZaZmclSUlJYeno6u+GGG9jmzZv57detW8fGjRvHUlJS2Oeff842bdrEEhISWG5uLr/N9u3b2dVXX83S0tLYuHHj2AMPPMBqampa/R1xHMemTJnCFi5c6NF+Nq8JjDG2fPlydtNNN7H09HSWnJzMxo8fz+677z6Wn5/fag3//vsvu+GGG9ioUaNYamoqmzZtGnvuuedYWVlZs1rPO+88dvfdd7e6r1Pt3buX3XPPPSwjI4MlJyezSZMmsQULFrCdO3d6bGcymdj999/PRo0axYYNG8YWLFjAtm7d6vGzbe13eOLnMXbsWJaQkMAOHz7cYh033XQTGzp0KEtOTmbnn38++9///teu50AI6VwCxrpo+BdCCCHt8u6772LRokXYs2cPZDKZt8s5Z/X19fDx8YFEIgHgHoQiIyMDM2fObHEQDdL7/PTTT3jmmWewePHiVs9A9QSLFy/Gfffdhz/++OOMI1kSQkhLqBseIYSQTlNQUIAJEybgySefREFBAQoKCvDss8/CYDDgsssu83Z5pJNccsklSElJwcsvv+ztUlplNBrx5ptvYs6cORSUCCFnjcISIYSQThMXF4cPP/wQhYWFuPzyy3HllVfi0KFD+Oijj9q8ZoT0LiKRCIsWLUJeXh4+++wzb5fTokcffRShoaF46KGHvF0KIaQXo254hBBCCCGEENICOrNECCGEEEIIIS2gsEQIIYQQQgghLaCwRAghhBBCCCEtoLBECCGEEEIIIS0Qe7uA7lRT0+TtEgAAQqEAfn4q1NebwHE0vgY5MzpmSEfRMUM6io4Z0lF0zJCO6InHS2Cgzxm3oTNLXiAUCiAQCCAUCrxdCukl6JghHUXHDOkoOmZIR9ExQzqitx4vFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFFJYIIYQQQgghpAUUlgghhBBCCCGkBRSWCCGEEEIIIaQFXg9La9euxbhx43Dvvfe2uR3HcXjzzTeRlZWFUaNGYd68eSgpKemmKgkhhBBCCCH9jVfD0ieffILnn38e0dHRZ9z222+/xV9//YWPP/4Yq1atQkxMDO644w4wxrqhUkIIIYQQQkh/49WwJJPJ8PPPP7crLP3www+YO3cu4uLioFarce+996KgoAC7d+/uhkoJIYQQQggh/Y3Ymw9+/fXXt2s7q9WKI0eOICkpiW9Tq9WIjo7G3r17MXTo0HbtRygUQCgUnE2pnUokEnr8S8iZ0DFDOoqOGdJRdMyQjuorxwxjDA4nB4eTg4tj4DjG/2t3cbDanOA4Bo6x4//i5PKJtlPu42rhtt5oA2OATCJ0t7OT6wvKDIgIVIHjGJzMAQezw8lc4JgTdphRUm1EkK8cEDK4YINLYAMgBAMDYxwYONildRC5VMd7XDF4/j+D0eKAQiYCBBycilqIbBq4+2ad+H/G34aANV+n0AOcCJxDAggAAXD8/0708GKnLbv3AwGDQOyAwAGMyJchqbAC1vNmIGv25V32++xsXg1L7dXY2AjGGLRarUe7VqtFQ0NDu/fj56eCQOD9sHSCRqPwdgmkl6FjhnQUHTOko+iYIR11pmOGMXcocDo5mKwO2B0cnC53MGk02iAUCPhljmOoN1ghlYj4+7k4Bs7Fob7JBo5jkEpEcDhdcLoYbHYn7A4OdocLeYV1iAnVNAs8+UX1CPVXQSwWwOVicB7fX22j9Ryf+fEwIDcBAs59W8BBoDBCILFDILNAILGBOSXH17kDidC3BszsA6HKAOYUAyEMZQIOAmErl5aogcpzrFTsAzhOWXZKzB3fiZCDUOw483anUZoliCvSwCKVoNyfQXr0AHx9VR1/fC/pFWHphHO9Pqm+3tRjzixpNAoYDBa4XJy3yyG9AB0zpKPomCEdRcdM/3AigFhs7pDBcQxOzh1UTBYHXC6GGr0FQqEAxdVG+Cgk7sDi4nCkzACxSIAmswM+Sgk4xiAQCGF3OKE32lFdb0aIvxIuF4PDxcFic/Jna7pLXSsBqKLOBAhcgNAFgcgJSBwQ+ZvcKwUchKpG9zqpO9wIpFYIVY0QCBmYSwSPMycCHA8+Z0+gMrj/FTvPaT8AAHb8s62AQcK5Q4jg+P+OFw0BBHC6GCQiIQQCAcyCBuhYuHudQHB8C/D/nmw72e6CHRw4SJxayCQiiIRC91oBPLY7cWJCyAHCqiaIaprAHT/56JRJkD1nPhoaTOf+vDtBe0JbrwhLOp0OQqEQer3eo12v18Pf37/d+zlxmrSncLk4OJ30hkTaj44Z0lF0zJCOomOmZ+A4BpvDBavdBbvTBYeDg9HicHcZczFY7U6YrE4YTHbojTYoZWJYHS4UVRjgcDKIhAJIxEJY7S6U1hjhp5Gh3mDr8ror6s7ijEWHuM/mqNWASMJBIgLEEgFEEhdc8nrUm4zQ+NvAhA64hDaImRRM4IJZVOvuIXYW35kLRK4O30cqlEAqkoFjLpidFsRqoiAQCCESCCEQCFFtrkGibzwabQYM9B0AkUAEocDdtU4kEMFProNIIAIA+Mp1EAqEEB6/v0QohVIs59t6Uq+pE+rqapGTsxh1dU0AALHTicFVtTj/xedhFsl71WtMrwhLMpkMAwcORF5eHtLT0wEABoMBxcXFSEtL83J1hBBCCOnvXBznDjYODmarw91FzOFCk9kOxgC90QaHi0Oj0Q67k4NQAFhsLhRXNaG+yQqrzQWNWnq8q1onnG04zbkGJbVCApFIgEajHUE6BQxmOwZF+0IuE8Pl4vgMUlVvQXyEFhKREGKxEAqZ+wyEWCSASCiAXCp2n5U4vmxz2QCJFUanAUeNRyAVScHgQrGxGGXmcgQpAuFiLtRYaiGAACeupnEd/89+Wp0iLXDqOQuPZ92OTKGWqGB0mJCgi4NQIIRUJIVEKEa4OpQPJoLjZ0+EcC87OAeifSIhFoogEoggF8sQqAjokSGmO+Tl7cG6davgcrlDpp+hCUnHShE37/8gC/CHuYecVWqvHhuWqqqqcMMNN+CTTz5BZGQkrr76anz88ceYOHEigoOD8Z///AeDBw9Gamqqt0slhBBCSC9yoiuaO9y4YLQ44OLcF/k3mR1wuFyoa7TCanfx3chcHIPTxeFQiR7hASrsKagDxxjEIncQsNg6fvbhdI3G0z/6n9mJLlMcYwjQyiERC1FRZ0ZcuAZmqxMxIT4Qi4So0VsQH6GD1e6Ev0YOhUwMxhg0KilEQqE7vAgEEIkE0KllUMjEkEtFELcxeINYLISvrwoNDaYWzxQwxmB2WlBv1eNoYxHy6w9BBSUOVh+BRChGtaX2jM+v0lx1cn/oWO+gEGUQmhxGJPrGQyQQocGmR3rwcCglSmikPpCIxNBIfSATySAWiiER9tiPxb2KXK6Ay+WCkDHEl5YjsqYOuimZ0I4Z4+3SzopXj4oTQcfpdH+DkpOTAwDYu3cvHA4HCgsLYbe7Xziuuuoq1NTU4LrrroPJZMLo0aOxaNEi7xROCCGEkG7DGEOTxYFGox1WuxN1Bqv7DI3D5THyWHFVE3w1cvdF/ccaEOyrwIFiPUL9lXz3MIlYCKeT6+DHbk/VDRb+tnsUtY7dX3g83EQEqqGSiyERC2GyOqFWSBAb6gOJWAiDyYHwQBXEIgGEAgG0KimEx8/MnAgxaqUEUnHndMPiGAcX4+DinLA4rTA4G1FvdsLqtKLCVA2RQAiOce7R38DhmKEEWrkGTOTCvsqDCFIEHl/PoaCxEA7u3M6OBcj94OCcaLQbMDF8LMRCMRptBkT4hEEukoHB3dVNI9NAdLw7mkwkRZgqBBKR5Jx/HuTsxcUNRJp/MJRr10BttUEaHoGgK6/xdllnTcD60ayuNTVN3i4BwJm/iSHkdHTMkI6iY4Z0VHcdMw6nC/UGG8w2J4wWB2x2F+oMVlTVm2FzuFDVYEFZjQkhfkocq2qCWOS+MN1bJGIhlHIxlDIxREIBREIhiqubkDbAHw1NNkQEqSEUCBARqIJSLoFMKgLHMejUUkjEIoiEAqgV7naZRAiJWNSt9XOMg5NzwsVcMNiNMNgMMNiNqLc2wGBvgpNzYk3Zxm6tKUYTBYvTArVEDalIgvSQ4VBLVAhWBsFPruu33dd6I6vVgtzcFYiLG4j4+EQAgK2kGMeeexrgOIh8NIh9+TUIZbIe+b4UGOhzxm3ofCMhhBBCzorF5h5goLLeDLuTQ1mNEXKpGAVljfBRStBosmPn4TN3tWrJsSr3F5xtBSWZROQ+8yIUHA8yAtQZbFArJAjxU6K6wYwBYVo0GG1IjNQBcJ8JCvZVQCwWwmJzIshXCQEAjUoKiVgIiUgIiUQIjVIKuVTUaz64Ozgn9NZGHDMU45D+KBqseuyvP9hljyc8fqZJIZZDK9Og0lSNJL9EfhACF3PBzjkxyDceGpkGYapgRKjDIBJ2b1gkXae4uAgrVy6F2WxCaekxhISEQVxXh+Lnnua3CbpmDoQymfeK7AQUlgghhJB+zulyj7RWWmdBaWUjympMOFbZBKeLg0wiQmmtCRzHoUZvdZ8hEQthtjk7/YyP4PgQxDKpO6wkRvmC4xhUCjHUCgmCfJXwVcugkIsR4qeASNi7J0NtD7vLjoLGIpQ2lcPuck9W2mDVo96qh1QkwcGGI+BYx7+lFwqE0Eh9IBVKIBKKUGGqQor/IAwJTD0+YpsAvjIdxEIRZCIZAhX+/AAHQoH7594TzxSQrudwOLBp01rs3buLb4uLS4DYxaHsrTf4Nr9ZF8JnVLoXKuxcFJYIIYSQPszh5GC2OaFvsqG4ugkFZQY0me3YfaQOfhoZmo53hWsvm93V7u0DtHJEBKrRaLJjYIQWDieH8ED3BPEBWjk0SimkEiF8lFIo5WIIe8lZnM5Sa6nDvroDcLgcqLHUweq0QiKSYG/NfoSoglHQWHjOjyEXyTEqZBhiNJEQCUSQiiSI8omAVqbhQw8h7VVdXYmcnMXQ6xsAAAqFElOmTEV0VCzK3nkTrib3/FHKlFQEXHixN0vtNBSWCCGEkF6E4xiMFgfqm6wwmh1oMNpQWWeGXCrC/qIGlNYYoVPLYHO4UNvKBJ0nnGn9CckxvoBAAKPZPeiAi2MI9VNCpZDA10cGrUoKH5UUPsevzelvoactjDE0OYyoNFWh3FiFAw2HsLc2n+/G1prWgpJCLIdcJAcDg97WiMF+Cai11GGgbgCClIEIU4cgyicCPlJ1Vz0l0g9xHIcdO7Zg27ZN4Dj3cRsbG4dJk6ZCqVSibNHbMO/bCwBQDk5C+F33erPcTkVhiRBCCPEiu8MFq8OFmgYLmswOVNabUaO3QCYRwWC2w2Cyw2J3orLO3O75d9raztdHhkCdAjKJCBa7E0nRvlDKJdD5yBDop4JaJoRaLoFCRh8R2mJ3OVBprkKlqRrFTaUAAwz2JlSba2B12eBiHOqtDa3e//SgJBNJ4eCc4BiHaE0kyo0VGBKYgnJjJSZHZCDedwDfFY6Q7tbYqMe2bZvBcRwkEgnGj5+CQYOSIRAI0LR1C0y7dgIARDodwu68G4I+1EWWXgkJIYSQTsLYybl7DGYHGk022OwulNeZoTfaIBYKcaSsEUIBUFBu6PTH1yglcLg4WGwuJMX4IjJI7R6hTSJCoFaOsEAVgn2VLd63v19/UmepR17dQVicFjiZCxzjUGGsBABw4LC3Nh+AO9TYXB2fD6klY0JHItE3Hukhwztlf4R0FV9fP4wZk4GjR48gK2s6tFodAMBlNKLyi0/57UJuvLnXD+hwOgpLhBBCSDs4nBzqDVZU6y2o0VtQUWdGk9kOq92FPQV1EIuEcLo6N2Ro1VJEBKqhkIqglIuhUkhgsbkQHqCCTi1DgFYOP417AtG2Jg/trxhjaLQb0GQ3wsm54GIu1FrqcKihAHpbI8qNlfCRqlFuqmz3PtsKSnKRDMn+g6CSKCESiCAUCgEGiIQiRKjDEKeLgUbqQ2eHSI/nHuGuGAkJg/m2IUNGIC1tuPu4hvvvq/y9d8COz4kaMu//oEpO8Uq9XYnCEiGEkH6LMQani0NtoxVHShtRXGWE1e5EaY0JAGCyOtp9XU9HgpJWLYWPwj00ta+PDMMGBoABCPZVIkArh1ohgVBI1/20hmMc7C47XMcnQbW5bNhTux8mhxkc43C44ShKjeVwcI4z7qvJYWzWJhVKIBVJIRQIYXFa4WIuxGljUG/VI8InDLGaqOMToYoQrYlEgMKPrhEifUZBwWHk5i6HzWaDRqNFSEgYAEAgEHgMpd+YuwqWw4cAAMqkZPiMGeeVersahSVCCCF9BscYGo126I02VNab4XBysDlcKKowwGhxHu8eZ0dFnRkAIBQIwJ3j3OwquRixYRr4+cjgo5RCLBIi1F8JiVgImcQdhjQqKZQyca+Zs6cnKDNWoN7agM0V21HSVAZfuQ6VpuoWw825CFUFQyfTIkgZgGifSAwLSoVUJO3UxyCkN7DbbVi3bjUOHMjj2yoqyviwdCrL4UOo/u5bAIBQrUbobXf22dc3CkuEEEJ6NI4xmCwOVOstaDK5R4GrbbSioKwR9QYbVHIxiqvP7gN0a0HJXyOH3miDr48MiZE6GC0OBOgUSIzUQaOSIkArh4/SPYkpOXsc41BhqkKZsQJV5hqYHRYcbSxCqbG82ba11vp279dXpoPZaYaLcZgYPhax2mj4yrQQCUQQHJ9fSCNV99kPd4R0VHl5KVasWIKm40N/q1RqZGVNR0REVLNtrceKUPbOm4DLBQgECLv1DogUiu4uudtQWCKEEOI1Lo6D0exATaMVFXUmlNWY0NBkg97o/q9Gf+YucHXtGCdBJBTAxbmDUXy4FgaTHUF+CmiVUgwI00AiFiEmxAdhASrq/tYJnJwTJU1lsLpsMDssKGw8hhJjGRwuJ441lUAhVsBxfILVjhigjYa/3B9WlxUhyiCEqoLd3eGEIshEUsTrBkBGZ4UIaTeXy4ktWzZi586tfNvAgYMwYUIm5HJ5s+0tRw6j5OUX+OXQW++ActDgZtv1JRSWCCGEdDnGGBqabCitMeFIWSM25VXCZHXCYmvfUNinEwkFkEqE4DhAJhEiIVIHq8OF+DAtnBxDXJgGaqUEwb5KKGViCkDngDEGJ+eEg3OgwdYIm8uGY4ZSNNoMMDnMkIgkMNgMqDLXtHugBIvT0uq61IAkBMj94GQupIcMg7/cfT0QDYpASOdbvPgvFBe75/SSyWSYODEbAwcmtritvaoKZW+9zi/7XTAbPiNGdkud3kRhiRBCyDlxODk0GG0wWRyoN1hR12jFwRI9bA4X9hc1QCYRwebo2BkEXx8ZrHYXAnVyDIkLgEQsRIBWjlB/FYJ8FZBLRdSFqhNYnVbk1R2E0WGCnbNhbdkmhCiDkFd3EL4yHawuKyzO9g1w0RaFWI5YTTSUEgXEQjEabQZE+oRDKVbAybkQr4tFhE8YFOLm32QTQrpOWtowFBcXIiIiGpmZ06BW+7S4HWMMlZ99DM7qfj0IvfUO+Iwc1Z2leg2FJUIIIa0ymO2oa3RfI1RvsGLHoRrY7C4UVxvh6yNDQ5PtjPtoLSglROoQF6ZBiJ8SCpkYQb4KhPgpIZWIOvtp9GtWpxWFjcVochhR0lQGiVCCvLoDLV4XBAB1FvdEqg02fbsfI1gZBIChylyDALkfBvrGIcInDPHaWMjFcvjJdXRmiJAewGw2QaFQ8l82RUXF4MILL0dYWESbX0DV//MXrEcLAAD+sy/qN0EJoLBECCH9msPJoeH4gAl7CupQZ7DC5nCh0WhHyRkGTWgrKKkVEvgoJRCLhIgKVsPXR45gXwW0KinCA9XQqaV0ZqgTWZwWrCheC45x2FG9+/hw2nYYHaYO7efEhKuRPuEQQIBYbRSUYgVMDgtiNJFwMQ7BykDIxTKEqoIpABHSSzDGcOhQPtauXYmMjMkYPPjkfEjh4ZFt3te4exfqfv8VACBS+8B32vQurbWnobBECCF9nNHiwJ6CWhSUGWB3urD9YA38NXLUGayw2jvWPU4mFSEiQAWni0EpFyM+XAuHk0NYgAoalRRBvgoEaOU0QWonOzG5apmxEg6XHQWNRVhZshY+EjWMDhMYOjb8uVAghFQoRZwuBvG6WIwLTYdO6QNfXxUaGkxwOjt3cl1CiPdYrRbk5uagoOAwAGD9+lzExQ2EVCo7433N+ftRvuht94JAgJBbboWwhYEf+jIKS4QQ0stZbE6U1hhhMDlgMNlQWNmE4qomFFe1fmaorLb1Mw5yqQhD4wOgUUmhkIkRGaRGgFYOnVoGH6WEzgh1MYfLgUJDMcxOCzaUb0Fe3YFWt21pziG5SAalRIlQVTCiNZGwuWyIUIchRhMJnUwLiZB+h4T0F8XFhVi5chnMZvdrvkajRVbWjHYFJVt5OUpffxUAIJBIEH7vA1AmtDz4Q19GYYkQQnowF8fBanehocmGmgYLjlU1odFkR35RAwQCoKqh9VHFWiISChAfroWT4xCkUyLETwGlXILoYB+EBSihlEu66JmQllicVuSWbsAxQwma7E0oNBR3eB/pIcMRrAyCRCjGhPCxkIrod0hIf+dwOLBx4xrs27ebb0tKSkVGxiRIJGceXp85nSh/721+OWTe//XLoARQWCKEkB7B4eRQWmNEea0Jq3eVwWRxorLeDLFIAKerY12spBIhhAIBOAYkRGqRHOOHYD8lUmL9qHucl9ldduhtBtRZ6pFTnIsDDYfbfd8L42YgWBmIKJ8IiIQiyEVyCkaEkGZqa2uwbNnf0Ovdg7UoFEpMmTINMTED2nV/5nSi4IF7wBndZ6590sfAZ2R6l9Xb01FYIoSQbsQxhhq9BaXVRhSUGVDfZMWW/OpWt28rKMWFaeCjlEImFSFtgD/CAlQI9lNALhVDLBbS9SdeVG2uQYG+CH8XLoNYKEatpe6M95GLZAhXh0ElUSLBNw6JvvHQyHygFCtoIAVCSLtJJGIYjwed2Ng4TJ48FQqFsl33ZRyH8vff5YMSAITcfEuX1NlbUFgihJAuYHO4UNtoRVmNETV6C/YdrYfR6kC9wXbGiVhlUhFsdheig30QFazGwAgdREIBAn0VCNTKoVHRSHI9Bcc47Kndj//l/4xAZQCqzTUwtzHhaktmD5iOqdGTKRARQjqFVuuLiRMzwRjDoEHJ7X6/YC4Xan74DqY9J7vuxTz/EgTC/v3aRGGJEELOQb3BioYmG8pqTaiqN2NDXiUajfYO7UOrkoJjDBOHhGHUoCBEBKkhpDDUI3GMQ7mxEturd2NlyVo4uZPB19TG9UaD/RIgF8sRp42BQCBAsCIQgUp/6GRaiIX0VkwIOTuMMeTl7UFTkwFjx07g2wcNSu7wvqq+/ByGjesBALLIKEQ+/BiEsjMPBNHX0Ss0IYScgdXuREWdGaXVRhwo1qO4uglyiQgF5YYO7yssQIWpIyMQHeKDAK0CagVdc9KTmRxmNFj1yC1djw0VW8+4fZAiAMn+g2Bz2ZAcMBhJfgk0+hwhpEuYTEasWrUMxcVFAICIiChERkaf1b4a163hg5IkMAjh99xHQek4CkuEEAL3BKuHSvSorDdj8/4qVNabz3pffhoZ4sO1GJkYhCBfBfw0cihlYgiF9IG5J6s0VaNAX4hVpetQYapq9/2ifMJxc8r18Ff4dmF1hBByUkHBIeTm5sBqtQIAfH39oVAoOrwfxnFoWLoYtb/85G4QCBD56OMQ+2g6s9xejcISIaRfsdicaGiy4UhZI3K2lcLp4jocjAQC9xDcE9LCEBaggr9WjkCtHH4aORQyelnt6WrMdagyV6OgsQibKrbBYG9q933lIhmiNJGYPeA8xGii6IwRIaRb2Ww2rFu3CgcP7ufbhgwZgdGjMyAWd/z9p/LzT9C0aSMAQCAWI+KBhygonYbe1QkhfU6T2Y7CiiaYbQ7sL2yAi+OQf6wB+g5cSxQeoIJYLERsqAYquRhh/iqEB7r/E/Xzi117I7vLgWXHVmFxUU677xOriYafXIdk/0EIVQcjQh1GgzAQQrymrKwEK1cuRVOTuwu4SqVGVtZ0REREndX+6pcu5oOS2M8fobfeAcWA9g0v3p9QWCKE9FqMMRSUGbDrSC0aTTbsOlwLk7XtkeZOJRIK4OIYkmJ84aeRI31wEOLDtZBL6aWxt7K7HKiz1uP7g7+iyW6C1WlBYzvOHIWpQhCjicKI4CGI0URBLqa++oSQnoPjOKxevZwPSgMHDsLEiZmQyeRntT/D5k2o/ekHfjl0/m0UlFpBnwgIIb2GxebE1gPVqKw3Y8nm1kcea4laIcHACC1UCgkGRekQHqBGeKCKJmntxWwuO1aVrMWG8q2os9a3+35qiQrXJ12JEGUQ/BV+XVghIYR0DqFQiMzM87B48Z+YODET8fGJZ70vw+aNqPzkI3457I4FUMTFd0aZfRKFJUJIj2S0OFBYYUBpjRFlNSYUlBtQ3WAGa32OVsSGamCxOREXpsGIQUGICFRBo5RCKhF1X+GkyzhcDhxpLMSXed/B6DC16z5BigAk+SfC4rQiyT8Rw4PSqCsdIaTHY4zhyJGDiI9P5K+NDA0Nx3XX3QyJ5OxHUTXn70flpx8DAARSKSLufRCKgQM7pea+isISIaRHKKs1YcmmY9icXwW5VAyjxXHG+yhkIkwfHY2MlBD4ac6uKwLp2UqayvBP4XIc0R+FxWltc9tBvgMRogpCiCoYIcogxOliKBgRQnqdpiYDVqxYgvLyUpjNZgwZMpxfdy5ByVZagtLXX+WXg6+fS0GpHSgsEUK6ldPFobTGiPJaE/IK63GgWI+GJpvHNqcHpQCtHKH+KoT4KTFsYAASInU0DHcfZHPZ0WhrRJGhBOvKNqGgsajN7eN1scgIG42hgamQimi+KkJI78YYw8GD+Vi3biXsdveARAUFh5CWNuycR960V1Wi+Pln+GXf6TOhGTPunPbZX1BYIoR0KcYYDhxrwIodZThSqofBfOYzRmKREONSQjB0YAASI3U0HHcfU2upw6aK7TisL8ARfSF8ZTo02PTtuu9ViRcjxX8wfOW6Lq2REEK6k8ViQW5uDo4ePQwAEAgEGDEiHSNGjDnnoORsMqD4+WfAnO4BkFTDhiPg0svPueb+gj6BEEI6FWOMP2u0alc5qtqYw+jEaHSZw8OhlIsxJC4AsWEaCGnumj7F4rTg18N/Y3dNHkzO5sdDW0FJJ9PimkGXIdn/7C9mJoSQnqy4uBArVy6D2ey+FlOj0SI7ewZCQsLOed+cw47S114GZ7EAAPzOn4WAiy895/32JxSWCCHnxOZw4XCpHvuO1mN/UT1Ka9q+8F6jlGDqqEiMGhQEf62c5izqoxhjWF++Gd8d/LVd22dFTYRUKIVCLEe4OhShqhBoZT5dXCUhhHjXvn27sWbNCn45KSkVGRmTIJFIO2X/1d98BXt5OQBAMzYD/hdd0in77U8oLBFCOsRqd+Kv9UU4VtWE/KIGtDE4HQAgMkiNgRFaTB8dhQCtoltqJN3PwTmxrXInDjQcxvaq3WCtHBmxmihEayIxyG8gBuriaD4jQki/FhMzAJs3r4NQKMKUKdMQE9N5cx3V/vYLDBvWAwCECgWCb5x3zl36+iMKS4SQNtUbrMg/1oCSaiOOVhhwpLSx1W1TYv0QHeKDAK0c/lo5kmP86IW5D2uyG7G4KAe5pRva3E4iFOOi+PORETYaEiG97RBC+i+XywWO4/hR7dRqH8yYcSF8ff2gUCg75TEYY6j96Qc0LFvCt0U9+gQE1JPjrNC7FiGExxhDXaMVOw7XYtnWYtQbbGe8z8hBQUiK8cXowcE0EEM/UGmqwod7voTe1ggH52x1uwCFP8aGjsTI4GEIoIlfCSEEen0DcnIWw8/PH5mZ5/HtYWERnfo4DcuWeASlyEefhDT03K9/6q/okw0h/RjHMeQfa8DWA1XIK2xAnaHteWwAICMlBAMjdRiTFEyTvfYja0o34IdDv7e5TXbUJEyLngKVpHO+HSWEkL6AMYa8vN3YsGENnE4nqqsrkZAwGBERUZ3+WKa9e1D7y08AAKFShdD5t0ExoPO69vVHFJYI6Wcammz4a0MRDpXoUV7b9mAMof5KxIdrkT44GAmROkjEdAq/P3FxLvxTuBxLj61scf2wwFTEaqORGTmBulsSQkgLTCYjVq1ahuLiIgCAUChEevq4Tj+bBACWw4dRtuhtgOMgkMkR+dCjkIWHd/rj9DcUlgjp4xhjOFSix+LNxdhTUNfmtsMGBiDEX4mxySEI9VfSSHX9UJPdiC2VO7CrZh+OtjIp7PWDr0R6yHAKSIQQ0oaCgkNYvToHNpu714afnz+ys2cgICCo0x/LcvgQSl55kV8O/b/5FJQ6CYUlQvogi82Jfzcdw6ESPaoaLDCY7C1u56OUYProKEwcEgaVXNLNVZKegGMcNpZvxeKiFWecGPbe4bchXhfbPYURQkgvZbfbsXbtShw8uJ9vGzJkBEaPzoBY3PkfvS1Hj3oEpaA510M9dFinP05/RWGJkD6CYwzr9lTgr/WFqGtjYIaIQBX+b1YyIoPU3Vgd6UnMDgv21OZhffmWVs8enSCAAC+OfxwaKc15RAgh7SEUClBdXQXAPdpdZuZ5XXJ9EgDYqypR+tpL/LIueyp0kzO75LH6KwpLhPRyDqcL3yw9hHV7K1pcH+KnxHnpkRgSHwCdmua06a8ONRTgs33/hdHR9nVqKf6DMDxoCNICk6EQy7upOkII6TvEYgmys2dgz54dGD9+MmSyrnktNeXtQ+WnH4E5HAAAvwtmIeCiS7vksfozCkuE9EJGiwPr9lSgtMaIDfsqm62XS0W4+7I0DIzUQUjXlfRb5YZK7CjLx9d5P7a5XawmGvPTboCPlM42EkJIR9XW1mDPnh2YPHkqhMev9Q0MDEJW1vQueTzGGOr//Rt1v/3Ct/lOn0lBqYtQWCKklyitbsI/a4/i7w1FrW4zeWgYrsiMh1xKf9r9kYtzYX35FuyvP4C9tfmtbqeVajA7bjoG+yVAK9N0Y4WEENJ3cByH3bu3Y/PmDeA4FzQaLUaOHNPlj9uwbIlHUFIOTkLgZVd0+eP2V/SJipAezOni8PPqAizbWtLieqFAgMhgNTKHhSMjNRRCIZ1F6m9qzHX4LO+/MNia0Gg3tLqdVCjBvJQ5SAkY3I3VEUJI32QwNGLlyiUoLy8DAIhEIsjliq5/3M0bUfvTD/xyxMJHoExI7PLH7c8oLBHSA5VWG/HH+kJsP1jT4vpBUTpMHBqG9MHB1M2uH3JxLiw7thp/Fy5tc7tBAXEYGTQMY0PSu6kyQgjp2xhjOHgwH2vXroTD4R5pNjAwGNnZM+Dr69elj+00GFD5yUf8ctidd1NQ6gYUlgjpIcxWB5ZsKcH6vRVoaGo+ml16UgguGBeFcH8VzW/Tz5gdFlSZa/Dn0SU41HCk1e3SApIRrg7FlMjx0CrU8PVVoaHBBKeT68ZqCSGkb7JYLMjNXY6jR92vwwKBACNGjMaIEaMhEom69LE5mw1lb77GLwddN5eGB+8mFJYI8SKOMew4WIN1eytanDA2LlyD2FANrpmagMAAH/rg24+YHGasLduIv462ffYIAG5JvR5DAlO6oSpCCOm/Nm5cwwclrVaHrKzpCAkJ65bHrv/nL9hK3F3yNWMzoJ04qVsel1BYIqTbMcaQV1iPLQeqsW5Py8N9j0gMxNVZA+GncQ83KhYJu7NE4iUmhxlf7/8B++paH5wBAOK0MUgJGIz0kOHQybTdVB0hhPRvY8ZMwLFjhRgwIB7jxk2CRNI9k7lXf/8/6HOWAQAkwSEIvnEe9TDpRhSWCOkGLo7D7iN1WLWzDEfKGmGzu5ptEx+hxYzRURgaH0Avgv1IsaEUPx76A4WGY61uEyD3w4zYbESowxCmDoFQQOGZEEK6WlVVBbRaHT9wg1KpxNVX39AtAzmcYNi4gQ9KABB26+0QCOk9oDtRWCKkCxVXNeHzf/JRXG1sdZtZ42IwdVQk1Iru+YaK9AxH9IV4c8cHra5XS1SYHpOFSRHjKBwRQkg3crlc2L59M7Zv34wBA+IxbdoF/JeY3RmUjLt2ovKzj/nloGvmQBYZ1W2PT9woLBHSyVwchx9XFmDHoWrUGZoP1OCvkWHysHBkDo+AQkZ/gv2F2WHBlsodWFmyFnXW+ha3GeQ7EBnho5EakASJkI4NQgjpbg0N9VixYjGqq6sAAMXFx9DUZIBG071dno17dqF80dsAAIFYjKA510M7fmK31kDc6N2YkE5UXNWEZ77cCsY82+PCNIgL1+LSSQMgEXftiDmk53BxLryw5U1Umavb3G7O4CswJmQEdb8khBAvYYwhL283NmxYA6fTCQAIDQ1HVtb0bg9KlqMFKH/nLX45+IYboRmb0a01kJMoLBHSCXYdrsU7v+xp1n5lZjxGJwVDp5Z5oSriLRzj8Pr291FkKG51m6lRkzEkMBmx2uhurIwQQsjpTCYjVq5cipIS97WjQqEQo0dnYMiQERB28/VBtvJylL31Br+snTiZgpKXUVgi5Cy5OA4b9lXii38PNFsXEajGI3OGUze7foYxhtWl6/Hz4T+brYvyCcfwoCGYED4WcjGFZ0II6Qnq6mrw++8/wWazAgD8/PyRnT0TAQGB3V6LpeAISl56nl8Ouu4G6CZN6fY6iCf6JEdIB9kcLuw+UotP/94Pp8uzv51KLsbCa4YjMkjtpepId7M4LcirO4itlTuwr655cFZLVHhw5AIEKLp2ZndCCCEdp9P5QaPRoqbGiiFDRmD06AyIxd3/8dheXY3S117ml/1nX0RBqYegsERIOzWa7Fiy+RiWbilpti4uTIObzh+MUH+VFyoj3vBt/s/YULGlzW3uHnYLEnzju6kiQggh7eFyOSESuT8Ci0QiZGfPgNlsRHi4d0aaM+3bg4pPPgI7fq2UfMAA+M++yCu1kOYoLBHSBrPVge9yDmP9vsoW11+VGY9JQ8Mhk9KgDf0BYww7qvfg87xvW90mUOGPO4f+H51JIoSQHsbpdGLz5vUoKyvGpZdezQcmX18/+Pp65zXbVl6O8kXvuIOSQICAiy+F38wLvFILaRmFJUJacKSsEf9uPIZdR2qbrQvQynHZ5DiMTAyCUEijl/UHhxqO4PN9/0OTo/l8WT5SNa5IuAiJvvFQSZReqI4QQsiZ1NbWICdnMerr3e/r27dvRnq6dwdOYE4nKj//hD+jFHjlNfDNnurVmkhzFJYIOY4xhr83HsPW/GqU1jT/UJwW54/Jw8IxJM6fhnjuBxhjKDIU46M9X7UYkgDg4VH3INInrJsrI4QQ0l4cx2H37u3YvHk9OI4DAERFxSA5eYhX62Ich8N3zAdcLgCAz5ixFJR6KApLpN/jGMPqnWX477JDzdb5a+S4dmoC0uL9IaSA1C/YXQ68teNDHGtqfm0aAET5ROCm5GsRqPTv5soIIYR0hMHQiBUrlqCiogwAIBaLMW7cJCQnp3n1S0/GcSh78z98UBIqlQiZO89r9ZC2UVgi/RZjDHmF9fhxVUGzM0npg4NwZeZA+PrQEM/9QZPdiK/2f49GmwHlppavT7tu8BUYEzqymysjhBByNg4cyMPatavgcNgBAEFBwcjKmuG1a5NOYByHmp9+gDl/P98W9fjTEHhhBD7SPvSbIf1OYYUBWw9UY+nmYpw68LdCJkJGaihmjommSWT7AbPDgipzDb7O/x7V5ubXpgFAiv9gXDDgPOpqRwghvUxpaTEcDjsEAgFGjBiNESNGQyTy7mBMjDEUPvIgnHV17gaRCHFvLYJIofBqXaRtFJZIv1HVYMajH20Ca2Hd1JGRuHTSAEglNKpdf/D1/h+wuXJ7i+ukIiki1WG4Z/itEAq6d+Z2QgghnWPChCkwm00YPToDwcGh3i4HjDFUfvLRyaAEIPqpZyko9QIUlki/cOBYA179bqdHm1AgwJVZ8RibHAK1QuKlykh32l61C5/n/a/FdRlho3FV4sUUkAghpJdxOBzYsmUDhg4dAZXKPSm8TCbH7NmXebkyN8YYKj76AMZtx+fmE4kQ+9JrkPjRFBO9AYUl0qcxxvDFvwewbm+FR/uz89IREaj2UlWku3GMw4JVD7e47pFR9yBYFQSJkF4OCSGkt6mqqkBOzmI0NurR0FCH88+/uEeNWMsYQ93vv54MSgCin3qOglIvQp8OSJ+1akcpvjlthLtxKSG4+YIkL1VEvOGYoQSvbnu3Wfu8lDkYHpTmhYoIIYScK5fLhe3bN2P79s1gzN3BXiwWw+l0QiLpGb1FOKsFZW+/Ccth92cRkdoHUU89C4mvr5crIx1BYYn0KVa7E/9ddggb9jUf0eySiQNwwbiY7i+KdDuOcdhauRNf5//QbN1ViZdgQvgYL1RFCCGkMzQ01CMnZzFqaqoAABKJFBMmZCIxcXCPOatkr6lG6WuvwFl//BolgQARDz5MQakXorBE+oSGJhuWbinG6p1lsDs5j3WZw8Nx0YQBdF1SH8cYw+6afcgpXoNCw7EWt7k4/nwKSoQQ0ksxxrBv325s3LgGTqcTABAWFo7MzOnQaLReru4ke3U1ih5dyC8LpFLEvPAKBaVeisIS6dUYY1j0617sPOw59LNCJkJcuBa3zEqmkNSHuTgX1pZtwu8F/8LBOVrd7prES5ERProbKyOEENLZjhw5iLVrVwIAhEIRRo/OwJAhwyEU9pyBeTi7HaWvv8IvS4JDEP3EUxDKadS73orCEum19EYbFv26F0fLDXxbgFaOKzMHYkRioBcrI13Jxbnw19GlWF68us3tJoaPw8zYbPhIaSAPQgjpC+LiErBv327YbDZkZ89AQEDPeq/nrFZUfv4JPzy4dnImgudc7+WqyLmisER6nSazHb/kFmDN7pMj3AkEwNVZA5E9MtKLlZGuVKAvwhs73m91vUggQoJvHC5PuBDByp71BkoIIaTjbDYrzGYzfH3dI8cJhUKcd94FkEplEIt71kdYR30dyt74D+yV7s8m8gEDEHTNHC9XRTpDzzrSCGkDYwzfLDuE1TvLPNqHxgdg/uxkyKQ0oWxftaxoFf44urjFdbMGnIfJEeMhF8u6uSpCCCFdpbS0GCtWLIFYLMYVV1zHj3CnVKq8XFlzxl07Uf7eO8DxUfkUCYkIu30BBD2oeyA5exSWSK9gsTmx6Ne9yD/WwLdpVVLcOHMw0uL8vVgZ6UpLilbir6NLmrWPDhmBi+PPpy52hBDSxzidTmzevA67d+/g2woKDmHQoGQvVtW6ik8/QtOmjfyybup5CLziqh4zKh85dxSWSI+363At3vllj0fblZnxmDYqkl6M+iDGGPbXH8L7uz9rti5BF4e7h8/3QlWEEEK6Wm1tNXJyFqP++HDbMpkckydnIy4uwcuVNcc57Ch783VYDh0EAAjEYoTcPB8+I0d5uTLS2SgskR7LxXH44Pc87DhUw7f5a2R46JrhCNDRqDJ90f66g3ivhZAkEohw25AbMdiv571hEkIIOTccx2HXrm3YsmUDOM49/UdUVAymTJkGlapn9SBgjMG4bSsqv/gUzG7n2yMWPgrFgAFerIx0FQpLpEeyO1xY+MEGGMwnh4M+f2w0Lpk4gM4m9UFlxgq8uOXNFtfdOfRmCkmEENJHuVxO/PnnL6iocF+PLBaLMW7cJCQnp/W493vOZkPlZx/DuGM73yby8UH0U89CrKM5lPoqCkukR7HandiUV4Vvlx+Ci3NfKCmTiPDMTaMQ5Kv0cnWkszHGcF/u47CfNkeSWCDCw+n3IFQV7KXKCCGEdAeRSAw/P39UVJQhKCgYWVkz+NHvehJnYyPKF70Na+FRvk07cRICr74WQonUi5WRrkZhifQIHGP4+M88bMmv9mhXysR44ZYx0KrohagvMTnM+GTv1zisP9psXXbUJFwcf74XqiKEENIdOI7zmEh23LhJ0Ol8kZIyFCJRzxvZ1mWxoOSl5+GodV8WoB6ZjpC5N9JEs/0EhSXidTa7C6//uAtHShv5NoVMhNkZsTSIQx/0z9Fl+Lcop1n7RXEzMTV6cvcXRAghpNsUFR3F+vWrMWvWpdBotAAAiUSCIUNGeLmylnEOB4pfeIYPSn4zL4D/RZfQsOD9CIUl4jWMMWw9UI2fVh1BncEGAFDIxLhoQiwmDw2HREwvRH1JmbECr257F07O6dE+yHcgrk+6ClqZj5cqI4QQ0tUcDjvWr8/F/v17AQArVy7BhRde0SO/EGUcB5fRCEdNNaq//QaOykoAgGJgAgWlfojCEvEKjjHMf201f10SAMSFa3DP5UOgkku8WBnpCm9sfx8FjUUebQO0Mbhn2HyIhD2vywUhhJDOU1lZjpycxTAY3D1IlEoVhg1L7/agxDgOLoMBTr0ezkb3f67GRjj1DXA2NsKpP75saARcLo/7yqKiEbbgHgpK/RCFJdLtqhrMeOzjzeDYyaB08YRYnD8uBsIe+A0TOXtV5ho8u+m1Zu3XJF6KjPDRXqiIEEJId3G5XNi2bRN27NgCdvw9f8CAgZg0KRsKRedd78OcTjgNjXDqG+Fq1LcQho7fNhiAUz57tJcyJQ1ht98JoZSun+6PKCyRbnXgWANe/W6nR9tj141AXLjWSxWRrvLJ3q+xq2afR1uK/yDMT5sLoYC+mSOEkL6soaEeOTmLUVNTBQCQSqWYMCETCQmD231GiXPYPcKOs7ERLr3ec7lRD1dT01nXKZDJIdZpIdbqINZqIdL5Qqw9vqzTQezrC0lwSI/sLki6B4Ul0m2WbinGDyuP8MsquRhv3Dmerk3qg/4pXN4sKM1PvQFpgcleqogQQkh3Kisr4YNSWFg4srJmwMdHA8A9XxF/tufUbnCNerj0jXA2NsCpbwRnNp314wsVCoh1vhDxwUcLsdYXIj4YudtoRDtyJhSWSLf4dtkhrNhRyi9PHhaO66Yl0Dc1fQxjDHeuesijbYA2BvePuN1LFRFCCPGGhKBQFKh84M8EGFBdD/2H76P2eDjiLJaz3q9QrebPAol1OohOCT58GNJoIZTJOvHZkP7Mq2GprKwMzzzzDHbv3g2lUomZM2fi/vvv9xh7H3CPx79o0SL8/vvvaGhoQEREBG677TbMnDnTS5WT9mKM4dmvtuFY5clT5LdemIz0wTTZaF/i4lz4dN9/sac2z6M9yT8RdwyZ56WqCCGEdJcjRw5CoVAiPDwS5vz9KFv0DgbZrBAAMJ7pzgIBRGqfk+FHd7IbnOjUYKTRQiihQaBI9/JqWFqwYAGSk5ORk5ODuro6zJ8/HwEBAbjxxhs9tvvuu+/w008/4auvvkJ0dDTWrFmDO++8EwMGDMCgQYO8VD1pj3mvrPJYfmTOcAyM0HmnGNLpjjYew7cHfkalqarZuhkxWbhgwHleqIoQQkh3sVqtWLr0Hxw8mA+12gfZSi2MS/4FAAgAiHS6U84EHe8Wd6LtRBjy8YFATJ2dSM/ktSNz7969OHDgAL744gv4+PjAx8cHc+fOxVdffdUsLOXl5WHEiBEYMGAAAGDKlCnQ6XQ4ePAghaUeijGGN3/c7dH27Lx0RASqvVQR6SyMMeypzcPHe79udZv7ht+OOF1M9xVFCCGk25WUFGPFiiUwGAwAAJdej8ot26EGAKEQofNvg8+IUV6tkZBz5bWwlJeXh/DwcGi1J0dBS05ORmFhIYxGI9Tqkx+qJ0+ejKeffhr5+fmIi4vD2rVrYbFYkJ6e3qHHFAoFEAq9f42MSCT0+LevsTtdeOXbHThc2si3PXTtcMSEarxYVe/WU46ZcmMlntnwnxbXqSRKPDf+Yagkym6uirSkpxwzpPegY4a0l9PpxIYNa7Fr13a+LdxixcCDRyDmOABA9EOPQJWY6K0SSQ/UW19jvBaW9Ho9NBrPD88nglNDQ4NHWJo2bRry8/Nx0UUXAQAUCgVeeeUVhIaGdugx/fxUPWpAAY2m743AwhjDFY/+A6vdPZmbn0aGp24eiwE0NHin8OYx8+6mL7D22JZm7U9PuRcJAXEQ0+SyPVJffJ0hXYuOGdKWyspK/Prrr6ipqQEAyCUSDDx4GMF699klia8OQ996A1Idve+TlvW21xivdhBl7ZwY7Pfff8fvv/+On376CYmJidi4cSPuv/9+hIaGIi0trd2PV19v6jFnljQaBQwGC1wuztvldKr3f93rEZSevjEdOqUYDQ1nP/wn8e4x4+CceGLtS2iwNXq0P5vxEIJVgQCApkZrt9ZEzqwvv86QrkHHDDkTk8mIL7/8FC6X+30+VCZH/NYdkDmdAADl4CRE3XMvTEwME73vk9P0xNcYX1/VGbfxWljy8/ODXq/3aNPr9RAIBPDz8/No/+9//4srr7ySD0aTJ0/GmDFj8Oeff3YoLHEcA8d1fObmruJycXA6e8bB0hne/30fth2o5pcfv34k1ApJn3qO3tadx4zJYcYXef9Dfv2hZuvey3wVAOh32wv0tdcZ0vXomCEtYU4nXMdKEC9X4bCxEQml5QivqcOJr6ADr7oGuqyp4AQCcHT8kDb0ttcYr4WllJQUVFRUoL6+ng9He/fuRXx8PFQqz5THcRz/LcYJdru922olZ/b72qMeQemZm9KhU9McB73Vol2fthiSLom/AFlRE71QESGEkO7EGIO9ohyVu3ZAfOQIzAcPgNlsCBcI4C+VQGlzfw6T+OoQeNkVUI8e5+WKCekaXgtLSUlJSE1Nxeuvv45HHnkEVVVV+OKLL3DTTTcBAKZPn47nn38eI0eORGZmJn7++WdkZWUhPj4emzZtwsaNG/ltiXet3FGKP9cX8ctzpiUgMohGveuNjugL8eaOD5q1x2qicfuQG6GkwRsIIaTPcjU1wZSfB/P+PDTk52OfVoU6jQ9GHzwMlc0GABCJRAgYEA9lUgp80lIQmjYY+kZLrzpTQEhHePWapXfeeQdPPPEEMjIyoFarcdVVV+Gaa64BABQWFsJsNgMA5s+fD6fTiTvuuAP19fUIDw/H888/j7Fjx3qzfALgi3/zsXZPBb98xZR4ZA6P8GJF5Gy9sf0DFDQWerTFaKJw/4jbIRT0rpFrCCGEnBnncMB65DBM+/NgztsHW0kxwBhqtD7Ij4qA/fgEsEVxMRgfEQtlUjIUAxMglLl7jojFQgiE9P5A+jYBa+8oC31ATU2Tt0sA4H5x8fVVoaHB1Ku/iflfziHkbCvll5+4YSRiaXjwLtGVxwxjDHeueqhZ+z3DbsVA3wGd+lik+/SV1xnSfeiY6fsYY7CXl8GclwfT/n2wHDoIdsplDU6hEIcjQlEW4M+3JQ0cjIzJ2ZAcD06nomOGdERPPF4CA33OuA1Nl0zOyupdZR5B6d4rhlBQ6oUsTiseWPOkR9u40FG4dvDlXqqIEEJIZ3I2NsKcn3c8IOXB1ahvto1AIoElMRG7lDIYnQ4AgFKpQmbmNERFxXZzxYT0LBSWSIfV6i34ZulBfvmpuaMQHXLmZE56lkMNR/D2zo892v4v5ToMDUr1UkWEEELOFWe3w3L4EMz798G8Pw+2kpIWt5NFRUOZlAxVcgryjY3Ysm0T2PGgNGDAQEyenA25vHfNh0NIV6CwRDrExXF46+c9ONF589JJAygo9TIc4/DY+hdgsHt2S30x4wloZfS7JISQ3oRxHOxlpTDlucOR5fAhMIej2XZiX18oBydDmZwC5eAkiDUne4OoD+4HYwxSqRQTJmQiIWEwBALvz0tJSE9AYYl0yLfLDqG81j3RXFqcP84fG+PdgkiHLD+2Gr8X/Nus/eXxT8JHSiMYEkJIb+DUN8CU5x61zrw/D64mQ7NtBFIplImD3OEoKRnS0DA+AJ1+uXpCwmA0NTUhMXEwfHyoSz0hp6KwRNrtlW934GCJHgAQEajG/81K8m5BpEOKm0qbBaXbh8xDsn+ilyoihBDSHpzNBsuhg/yodfbysuYbCQSQRcdAlZQMZVIy5HHxELYwKIPR2ISVK5chLW0YYmIGHL+rACNHju7qp0FIr0RhibTLlvwqPigB7gEdVPLmL8KkZ/o2/2dsqNji0fbK+KeglqpauQchhBBvsldVwrh9G0z782A9chjM6Wy2jdjPH8rkZKiS3F3rROq2ewgcPnwQa9bkwGazoa6uBldddT0UCpo/j5C2UFgiZ7TvaB0+/COPX746eyB8fWRerIh0xKJdnyK//pBH27tTXqa5kwghpIfS565GzQ//8xjWGwAEMjmUgwe7B2ZISoEkOLhd1xZZrVasXbsShw8f4NsGDUqCVCrt9NoJ6WsoLJE2NTTZ8MaPu/nliybEYurISC9WRNrLxblw1+pHmrW/M/klCkqEENJD6XNXo/qbL/llWUwsVCmpUCWnQB47AAJxxz66lZYWY8WKJTCZjAAAHx8NsrKmIyyMJpAnpD0oLJFWHatswjNfbuWXRyQEYnYGzbfQG7QUlMLVoXg0/V4vVUQIIeRMzPn7UfPD//jloOvnQjdx8lnty+l0YvPmddi9ewffNmhQMsaPnwyplHqHENJeFJZIiyrqTB5BSSYR4baLUrxYEWkvxlizoHTPsPkY6BvnpYoIIYScSdOWzaj4+AN+OfLhx6CIH3jW+2toqMfevbsAAHK5HJMmTUVc3Nnvj5D+isISacbp4vDB7/v45awREbh2aoIXKyLt5XA5cE/uYx5tdw69mYISIYT0YKWvvwZz/slrg0P+79ZzCkoAEBgYhJEjx6CysgKZmdOgVNKAPoScDQpLpJkPft+H0hr3XEohfkoKSr1EkaEYr21b5NH2wIg7EKuN9lJFhBBC2uKor0fN9996BKWga+ZAM3pMh/fV2KhHVVUFEhIG823Dh6dDIBDQBLOEnAMKS8RDWa0JOw/XAgDCA1R47PoRXq6ItMcxQ0mzoPT8uEfhK9d5pyBCCCGtYhyHxtUrUf2//3q0h956B3xGjurYvhhDfv4+rF+/Gi6XC76+/ggMDAIACIU0mA8h54rCEuExxvDpX/v55QlDwiCX0iHS0zXaDHh127seba9PfA5yMV3ASwghPVHZ22/AnLfPoy3klls7HJTMZjNyc5ejsLAAgHty2erqSj4sEULOHX0SJrzXf9iFY1VN/PK0UTREeE/XZDfi0fXPe7S9M/kliIQiL1VECCGkLQ05yzyCUsQDD0GROKjDXeWKigqwatVyWCxmAIBO54usrBkIDg7p1HoJ6e8oLBEAwLo9Fdhf1AAAkIqFePGWjveXJt3rl8N/YWXJWo82mmyWEEJ6LkvBEdT89AO/POD1tyDW6jq0D4fDjvXrc7F//16+LTV1KMaMmQCJRNJZpRJCjqOwRGAw2/H5v/n88h2XpMJPI/diReRM3t35CQ40HPZoo8lmCSGkZ2KMoeb7/0G/YjnfFnrbnR0OSowx/PnnL6iqqgAAKJUqZGaeh6iomE6slhByKgpL/RzHGF7/fhe/nBTji9QB/t4riLRpZ9VefLj7q2btb09+kbreEUJID8RZrSh7501YDh10NwgECJpzA3xGjOzwvgQCAYYNG4UlS/5EXFwCJk3Kglyu6OSKCSGnorDUz32z9CBKqo0AgJRYP9x7xRAvV0Ras754a7OgdPnACzE5MsNLFRFCCGkL57Cj7O03YDl8CAAgkMkRcf9CKAYMaPc+jMYmqNU+/PKAAfG45JKrEBwcSkOCE9INKCz1Y8VVTcjdVc4v3zI7mV54e6h1pZvxzf6fPNpuTZuL1IAkL1VECCHkTMoXvcMHJcWgwQi79Q6I1Op23Zcxhr17d2LjxrXIzDwPAwcO4teFhIR1Sb2EkOYoLPVTjDG8/9vJ0XhunDEIagVdGNoT/XN0Gf4tyvFoey/zVS9VQwgh5EwYY6j95Sd+1DtpeAQi7rkfAnH7PnYZjU1YuXIpSkuLAQDr1+ciNjYe4nbenxDSeeivrp/akl+Nar0FADBsYAAmDKFvqXqij/Z8hT21J2d2j9FE4sGRC7xYESGEkDOp/ORDNG3ZDAAQ+fgg4t4H2h2UDh8+gDVrVsBmswEA/P0DkZ09g4ISIV5Cf3n9EMcYPvvn5Oh3N50/2IvVkJYcbjiKt3Z+6NEmForxyJi74XRyXqqKEELImZR/+D6M27YAAIQqFSIefARine6M97NarVi7dgUOHz7Itw0bNgrp6WMhEtHHNUK8hf76+qF1eyrgdLk/cI9IDIRKTt3vepLfjvyDnOJcj7ZLBp6Pq4ZfgIYGk5eqIoQQcia1v//KByUAiH3hlXZdo1RRUY5ly/6GyeQecMnHR4OsrOkIC4vosloJIe1DYamfsdld+HLxAX75xhl0VqmnYIzhtW2LcKypxKN9fuoNGB6a6qWqCCGEnAljDGVvvc5foyRUKhE6//Z2D+Ygk0lhtbq7xg8alIzx4ydDKpV1Wb2EkPajsNTP/LDy5ESmF0+IhVJOh0BPUG2uxTObPAdtGBE0BDelXOuligghhLRX9Tdf8kEJAgGiHn0C0pDQdt/fzy8AEyZMgVyuwIABA7uoSkLI2aBPyv2Iw8lh9SlDhc8cG+3FasgJLs7VLCilhwzHDUlXeakiQggh7WUrL0fj2jUAAKFCgYgHH24zKHEch507t8HlciI9fRzfnpSU1uW1EkI6jsJSP3Jq97uZY6IhEgq9WA0BgEpTNZ7b/B+PtlcmPAW1ROWligghhLQXZ7ej6svPAMbcZ5QeewrSkJBWt29s1GPFiiWorCyHQCBAZGQ0QkPDu7FiQkhHUVjqJ45VNmFjXiW/fOH4WC9WQwBgS+UOfLX/e4+21yY8A6VE4aWKCCGEtBdzOlH46EK49HoAgO/0ma0GJcYY8vP3Yf361XA4HACAoKAQKJXK7iqXEHKWKCz1E6/8bwd/+5bZSZCI6aySN3245wvsrc33aHtu3CMUlAghpBdw1Neh5KUX+KAkCQmB/+yLWtzWbDZj9erlKCoqAAAIhUKMHDkGw4enQ0g9PAjp8Sgs9QOLNx2D1e7il8cktd5FgHS9OktDs6C0aMorEAgEXqqIEEJIe5n27UHlZ5/A1dTEt8U8/XyLk84WFhZg9eplsFjcI93pdL7Izp6BoCB6Hyakt6Cw1MdxjOGn1QX88tt3jfdiNYRjHJ7c+BK/HCD3wzPjHvZiRYQQQtpLvzIH1f/7L78skMkQ9cjjLQYlp9OBNWtW8EEpNXUoxoyZAImE5jYkpDehsNTHvfTf7fzt1AH+8FFKvVgNeWrjKx7LFJQIIaR3MOfvR82PJ68zDb7+RmgnTmp1e7FYgszM87By5VJMmTINUVEx3VAlIaSzUVjqw2wOFwrKDPzybRcle7Ga/q3BqsfjG170aLtzyM1eqoYQQkh7MY5Dw5J/UfvbLwBjEIjFCL/7PigHJ3ls53K5cPjwASQmJvHdqiMjo3HttTdB3MKZJ0JI70B/vX3YL7knu9+NTgqGXEq/bm9gjDULSjNisjHYP8FLFRFCCGkPR00Nyt59C/byMneDSISwu+5tFpTq62uRk7MEtbXV4DiXx5xJFJQI6d3oL7iP4hhDzrZSfnnujEFerKb/4hiHBas8u9o9MfoBhKiCvFQRIYSQ9rBXVuDYM0+CHR/qGyIRQuffDlXSyV4ajDHs2bMTmzathcvlHkipqOgoBg9OpUF7COkjKCz1Ud8sPcjf9tPIIJOIvFhN/3V6ULo0/gIKSoQQ0kMxpxP2qkpYjhxB9Tdf8u3yuHiE33M/RIqT0zs0NTVh5colKCsrAQCIRCKMGTMeaWnDKSgR0odQWOqDLDYncneV88vP3pTuxWr6r1Ul6zyWH0u/D2FqGi6WEEK8jXEcHNXVsJWVwl5eBltZGezlpbBXVQEul8e2mrEZCJn3fx5thw7lY82albDbbQAAf/9AZGfPgL9/QLc9B0JI96Cw1Af9s/EYfzsjJQRKOQ1T2p04xuHFLW+iwlTFty0Y+n8UlAghpJsxjoOzvu54GCpzh6OyMtgrK052r2uFQCaDesjQZkFp27bN2LJlPb88bNgopKePhUhEH6kI6YvoL7sPOlSi52/fOHOw9wrph8qNlXhhyxvN2gf5DfRCNYQQ0j8wxuBq1LtDUVkZbOWlx/8tB7NZ27yvQCyGNCwc0rAwyMIjIA0PhywsHGI/fwiEwmbbDxyYiJ07t0AuVyA7ewZCQ8O76mkRQnoACkt9jNPF4UhZIwBAo5JCKKR+093F4rQ0C0pToybjoviZXqqIEEL6HldT02nd59z/cmZT23cUiSANDoE0LByy8HBIwyMgCwuHJCioxVB0gtPpPgMlFrt7aWi1Opx//sUICAiEVCrrtOdFCOmZKCz1Mad2wbtofKwXK+lfSprK8PLWtz3abku7ESkBdGaPEELOhstsdgeh8uNni44HJJfB0PYdBQJIgoJOhqKwcPcZo+AQCDo4jHd1dRVychYjMjIKEyZk8u1hYRFn85QIIb0QhaU+ZvXOMv72+LRQL1bSv5welBZNeYVGQyKEkHbgbDbYK8r5QRZOnC1y1tef8b5iP3+Ps0TS8HBIQ0IhlJ3bGR+O47Bjx1Zs27YRHMdBr69HYmISgoLo2lNC+hsKS32I0eJAo8kOABgaHwCxqPVuBaRzNFj1zSacfXn8kxSUCCHkNJzDAUdl5fEzRaX8GSNHbQ3AWJv3FWm1kIWdvJ5IevyM0alDeXeWxkY9cnIWo6qqAoC7+9348ZMQGBjc6Y9FCOn5KCz1ITnbSvjbU0dSF4GuZnFamwWlZ8Y+DB+p2ksVEUKI9zGXC47qqtO6z5XDXlUJcFyb9xWqVO4uc6ddVyRSd/3rKmMM+fl7sW5dLn+dUnBwKLKzp0Or9e3yxyeE9EwUlvqQP9cXAQBkEhEGx/h5t5g+zuFy4IE1T3q0PTzqHgQo6OdOCOkfGMfBUVfrHoq7/OQ1RfaKCjCns837CmTyU64nOiUUabVeOTNvtVqwYsVSHDt2FAAgFAoxcuRYDB8+CsI2Bn8ghPR9FJb6CL3Rxt+OCFR5sZK+7+M9X2F3bZ5H21uTX4RESH9OhJC+zbhnF4zbt/PBiNntbW4vkEggDQ073n3ueDe68OPDcveg7soikQgNDXUAAJ3OD9nZMxAURN3uCCEUlvqMX1YX8LdnZdAoeF3lUMORZkFpfuoNFJQIIX0aYwz1f/2Buj9/b3kDkQjSkFDIwsLcZ4mOnzWSBLY9LHdPIZFIkZ09A4cPH8TYseP5YcIJIYQ+4fUR6/dV8rfT4vy9WEnfxRjD2zs/5pdDlEF4cOQCyMU0zwYhpO9yNupR+ubrsJeevC5WmZIGeUzM8cEWIiANDu7wsNzeVFFRhvz8fZgyZRp/hiskJAwhIWFerowQ0tP0nlc20qry2pMT8SVG6rxXSB/3xo73PZafGPOAlyohhJDuYT6Qj8ovPoWzzt1FDQIBYl58BdLAIO8WdpZcLhe2bt2InTu3gjEGP78ADB06wttlEUJ6MApLfcCm/SfPKl2QEeO9Qvqw7w/+hqONJyf8fSz9Pi9WQwghXcvZqEf1d9/CuG0r36adNAX+s2ZDrOudI8PV19ciJ2cxamtrAABSqQwqFV3jSwhpG4WlPmDFdvdEtAIBkEyj4HUqxhjuXPWQR9vE8LEIU9PEhISQvsleXY1jLzwPV6MeACAQixF4zRzoJk72al1nizGGPXt2YtOmtXC5XACA8PBIZGZOh4+Pj5erI4T0dBSWerlGow0Wm3uI1hmjo71cTd/zwJqnPJanRU/BhXEzvFQNIYR0rbpNm3Hk1deB46FCPXIUgq6eA7FW6+XKzk5TUxNWrlyCsjL39VYikQhjxkxAWtqwHjUaHyGk56Kw1MvtKajjb9PADp3rrR0fwuqy8suzB0zHeTGZXqyIEEK6BuM4VP3wI+oW/8u3BVx6OfxmnO/Fqs7dhg2r+aAUEBCI7OwZ8PML8HJVhJDehMJSL5ezvRQAoJKLMTCid37z1xNxjMNh/VF++fYhNyHZf5AXKyKEkM7nMptQ/+8/MG7bCsfxa3kgECD4hhuhHT/Ru8V1goyMySgrK0VSUipGjRoLkUjk7ZIIIb0MhaVezOniUFJtBABo1TLqUtCJFqx6mL/tI1VTUCKE9DmO+nqUvPwCnPUneyiIfdQIvmYOVKPGeLGys1daWoygoGBIpe4pHdRqH1x77U2QyWiKB0LI2aGw1IvtK6znb5+XHunFSvqWZzf9x2P58dH3e6kSQgjpGsZdO1G+6G1+Wezrh8CLLkLsrOnQG6xwOjkvVtdxDocDmzatxd69uzBoUDIyM8/j11FQIoScCwpLvdiaXeX87fTBwV6spO8oM1agylzNL48PGw21hIaWJYT0HcadO1D+3jv8smbCRARffyMkEhEEvbCbWnV1FXJyFkOvd3+BeOzYUZjNJiiV9NpNCDl3FJZ6KY5j/OAOceEayCS97w2uJ3pxy5v87XGho3D1oEu9WA0hhHQexhgali1B7a8/821+My+A/8WX9spu3BzHYceOLdi2bRM4zn0mLDp6AKZMmUpBiRDSaSgs9VJFlU3gGAMAZA6P8HI1fcOne7/xWL5m0GVeqoQQQjqXo74OlZ99AsvBA3xb0LXXQzeld47w2djYgJycJaiqqgAAiMUSjB8/GYMHp/TK4EcI6bkoLPVSOdtL+NsDw2kUvHN1x8qFHssLRy6gN1xCSI/DGANntYIzm8GZzXBZzCdvm83gLGa4TKZm62ylJcDxL9hEWh3Cbr0dioEJXn42Z6eyshx//vkznE73HIPBwaHIzp4BrVbn3cIIIX0ShaVeau8p8ysF6BRerKR3Y4zhzlUPebRNi56CaA0NmEEI6VkMWzah+r9fgzObz3of2omTEXDJZRCp1Z1YWfcKCAiCRqOFXt+AUaPGYtiwURAKhd4uixDSR1FY6oUYYzBZ3d+o0bVKZ8/msuO+3Mc92i6JvwBZUb1/bhFCSN9hryhHza8/w7Rzxxm3FSoUECqUECqVECmP/6tQQqhSQj10OJSDBndDxZ3P4XBAIpEAAMRiMaZOnQmO4xAYSIMbEUK6FoWlXqisxsTfnjEmyouV9F7Lilbhj6OLPdpmxk6loEQI6VHq/vkLdb/9crJBIIB66HD4jBkDkVIF4amBSKHolaPZtcVut2PdulWor6/DJZdcxZ9B8vcP9HJlhJD+gsJSL5R7ypDhY5NDvFhJ7/Sfbe+h0HDMo+3hUfcg0ifMSxURQkhz1d99C/2K5fyydkomAi6+FKJ+MtJbRUUZVqxYAoOhEQCwe/d2DBs2ystVEUL6GwpLvVCdwcrfDtDKvVhJ79Noa/IISkKBEK9PfBZSkdSLVRFCiKf6xf94BKXePHJdR7lcLmzZsgE7d27l2+LjEzF4cKoXqyKE9FcUlnqhXUdqAbiDEo3Y1jGPrn+Ovx3tE4mFoxZ4sRpCCPHEOA61P/2AhuVL+bbYl16DJLB/dDurq6tFTs5i1NXVAABkMhkmTszCwIGDvFwZIaS/orDUyyzdUszfnjCEuo11xOnDgz848k4vVUIIIS0ruO8ucEYjvxxx/8J+EZQYY9izZwc2bVoHl8sFAIiIiMKUKefBx8fHy9URQvozCku9zLaD1fztLJqMtt1e3vKWx/J1g6+gs3KEkB5Fv2qFR1AKu2MBlIOTvFhR9yovL4PL5YJIJMKYMROQljaMXqcJIV5HYamXOVZ58o1UKadf35m0NI/S/6Vej6GBKV6qiBBCmrMWHkXND98BAARSKQa89iZEqv4xkAMACAQCTJ6cDZfLiXHjJsHPz9/bJRFCCACAZnHrReoarXC6OADAFVPivVxNz+fiXM2C0oyYbApKhJAegzmdaNq2BWWL3gFzOiGQShH1yBN9PihZrRbk5ubAYrHwbQqFEhdccAkFJUJIj0KnJnqRfYV1/O3I4N47+3p34BiHu1Y/4tE2e8B0nBfTP0aTIoT0fKZ9e1D52adwNRn4tuC5N0EWGenFqrpecXERVq1aCpPJBIvFjPPOm0Xd7QghPRaFpV7kn40nh7xOjNR5r5Be4JmNr3osP5Z+H8LUNCcVIaRnMO7cjoqPPgBzOvm2oGvmQJM+xotVdS2Hw4FNm9Zi795dfJtMJgfHcRD1scl0CSF9B4WlXqS20T2/klohgVhEPShbs+zYKtRa6/nlG5OupqBECOkx9KtWovp/3wCMQSCVQjclC7rMLEj8A7xdWpeprq5ETs5i6PUNAACFQoHJk6chNjbOy5URQkjbKCz1EhbbyW8fIwL7dl/2c2FxWvBHwWJ++bKBszEyZJgXKyKEkJPqF/+D2l9+AgCI1D4Iu+seKAb03cDAcRx27NiCbds2gePc19zGxMRh8uSpUCqVXq6OEELOjMJSL3G04mSf9uyRfbs/+7l4YM1THstTIsd7qRJCCPHUuDaXD0pif39E3PsgpCF9+6x3fv4+bNmyAQAgFkswfvxkDB6cQtcoEUJ6DQpLvcSuQ7X87ehgmqCvJY22Jo/ldya/5KVKCCHkJJfFgqovP4Nx+zYAgNjXD5EPPNwvJpsdNCgZ+fl7IRSKkJU1HVqtztslEUJIh1BY6iU27a/kb/tr5V6spOd6dP1z/O04bSxEQrpgmBDiPfaKcjSsXAHDhvVgNivfHjr/tj4blMxmE2w2G3x9/QAAIpEIM2deDLlcDqGQrrUlhPQ+FJZ6CZPVfc1SqD/18W5JtbnGY/ne4bd6qRJCSH/GOA6mPbuhX5kD8/48j3WyqGgEXnUNFPEDvVRd1zp69DBWr14OpVKNyy67BmKx+yMGXZtECOnNziksOZ1O/sWQdB2H08XfjqIueC3KLd3A374h6SrqD08I6VYukwmN69agcdVKOGpP+fJGJILPyFHQZWZDPiCuT7422e02rFu3GgcOuMOh1WpFSckxGumOENIndDjpcByHRYsW4bfffkNdXR327NkDi8WCl19+GY899hikUmlX1Nmv5RU28LeHxNPM5qeyuxy4N/cxj7bhQWleqoYQ0t/YSkugX5kDw6aNYHY73y7S6qCbPAXaiZMg7sPX6ZSXl2LFiiVoOj6xrkqlRlbWdERERHm5MkII6RwdDkvvvvsufv31V9xwww146623AABmsxm7du3C22+/jQcffLCza+z3KuvN/O2ECJ33CulhGGPNglKYKgRiIZ3tJIR0HeZywbhrB/QrcmA5dNBjnTwuHrqsbPgMHwlBH+554XI5sWXLRuzcuZVvGzhwECZMyIRcTtfVEkL6jg6/kv/xxx/44IMPkJSUhLfffhsA4O/vjzfffBPXX389haUuUFR5cthwnY/Mi5X0HA6XA/ecFpRuTZuL1IAkL1VECOnrXE1NaFybC/3qlXDWn5z4WiAWwyd9DHRZ2ZBHx3ivwG5it9vx228/oK7O3d1QJpNh4sQsDBw4yMuVEUJI5+twWKqvr0dSUvMPpNHR0WhsbOyUooin8loTf1vYB/u7d5TNZcd9uY97tL2Y8Ti0Mo2XKiKE9GXWoiLoVy5H05bNYM6TE4SL/fygm5wJzYSJEPv0n9cfqVSKgIBA1NXVICIiCpmZ50GtputpCSF9U4fDUlhYGPLz8zF48GAwxvj2DRs2ILCPDoXqbaU17rA0MpF+vi7O1Swo3T3sFgpKhJBOxZxONG3fBv3KHFgLjnisUyQOgi4zG+qhwyAQ9Y8pCjiO8xj6e/z4KQgJCUNSUmqfHLSCEEJO6HBYmj17Nu644w7MmzcPjDEsW7YM+/btw3fffYcbb7yxK2rs1+oNJ+fmkEr6x5tyWx45ZS4lAHhtwtNQSmhYWkJI53A26tGYuxr63FVwndJbQiCVQjNmHHSZWZBFRHqxwu7FGMOhQ/nYunUjLr74SqhUagDurnfJyTSYDiGk7+twWJo/fz7sdjveeecdOBwO3HXXXQgICMCtt95KYakLbD1Qzd+ekBbqxUq879v8n2FynBzs4vlxj1JQIoScM8YYrEcLoF+Zg6ZtWwHXyekaJAGB0E7JhHb8RIhUKi9W2f2sVgtyc1egoOAQAGDVquW44IKLvVwVIYR0rw6HJafTibvuugsLFixAfX09ZDIZ1Go1nE4nKioqEB4e3u59lZWV4ZlnnsHu3buhVCoxc+ZM3H///S3O8l1QUICnn34ae/bsgU6nw4033oi5c+d2tPxep7rBwt8e2I9Hwqs212JDxRZ++eL48+Er13mvIEJIr8c57DBu3YqGlTmwFRV6rFMmp0CXmQ1VahoELbwn9XXFxUVYuXIpzGZ3N3CNRosRI0Z7uSpCCOl+HQ5LI0eOxO7duyEQCODvf3LOH6vViosvvhhbtmxp496eFixYgOTkZOTk5KCurg7z589HQEBAszNUVqsVN998M6699lp8/PHHOHz4MB599FFMmDABcXF9e9K7VTvLAACRQWoIhf23X/gzm17lb48MHorsqElerIYQ0ps56uvRuHolGtfmwtXUxLcLZHJoMzKgm5IFaWiYFyv0HofDgdWrc7Bnzy6+LSkpFRkZkyCR0DyKhJD+p91haePGjdi4cSOcTifeeOONZuuLi4vhPGWUoDPZu3cvDhw4gC+++AI+Pj7w8fHB3Llz8dVXXzULS4sXL4ZarcbNN98MAEhLS8Pff//d7sfqzSRiIRxODnaH68wb91GLC3M8lm9IuspLlRBCeivGGCyHD0G/YjmMO3cAHMevkwSHQJeZBc248RApFF6s0ruqqirw7bdLUFdXBwBQKJSYMmUqYmL69peShBDSlnaHJalUiqKiIrhcrhaDilKpxAMPPNDuB87Ly0N4eDi0Wi3flpycjMLCQhiNRqjVar59+/btSEhIwCOPPILly5cjICAAt99+O2bPnt3uxwMAoVDQI87OiERCj3/b4nC639AjgtQQi/tfVxC9tRF/Fy7jl/8vbQ6kkr470WNrOnLMEALQMXMCZ7OhcdNG1Ocsh62k5OQKgQDqtCHwy86GKjmlX3a1O11FRRkflGJj45CVNQ1KZf+6Tot0DL3OkI7orcdLuz91jhgxAiNGjMAVV1yBH3/88ZwfWK/XQ6PxHO75RHBqaGjwCEuVlZXYtm0bnnvuOTz55JNYsmQJHnroIcTHx7c451Nr/PxUPWqIU42m7W8wK+tOzq80JCEIvr79703r+SUnz2KG+4Rg6uAML1bjfWc6Zgg5XX89ZqxV1ahcvARVy1fAaTTy7SKVEsHZWQiZMR2K0BAvVtjzTJ48EZWV5Rg0aBCGDh3ao94vSc/WX19nyNnpbcdLh7+iby0oOZ1OTJs2DStXrmz3vk6dp+lM2yUnJ2PWrFkAgIsvvhjff/89lixZ0qGwVF9v6jFnljQaBQwGC1wurtXt9hfU8LdDfeVoaDC1um1f9MGuL1HSWM4vP5J+d7/7GZzQ3mOGkBP64zHDGINpfx7qc5bDuGsXcMp7jCw8HH7ZU6EdlwGhTAYrAGs/fT0B3D+rfft2IyAgCKHHr88SiYS46qqr0NRkhV5vPsMeCOmfrzPk7PXE46U9JyI6HJasVivef/997Nq1C3a7nW+vqamB1Wpt456e/Pz8oNfrPdr0ej0EAgH8/Pw82gMDA5ttGx4ejpqaGnQExzFwXPsCWndwuTg4na0fLMWVJy889vORt7ltX7O5Yjt2Ve/jlx8YcScETNSvfgYtOdMxQ8jp+sMxw1mtMGxcD/3KFbBXnPyCBQIB1EOHQ5eVDUXiIAgEAnAAuD7+8zgTk8mIVauWobi4CFqtDldcMYcfvEEgEPSLY4Z0LjpmSEf0tuOlw2HpxRdfxMqVKzF69GgsWbIE559/Pvbu3YuAgAA89thj7d5PSkoKKioqUF9fz4ejvXv3Ij4+HqrT5rKIi4vDd999B8YY3y2grKwMEyZM6Gj5vUrurpNv+lpV/xmF6GjjMXyd/wO/PDJ4KGK1UV6siBDSE9mrKqFftQKG9evAWU5OsyBUqaCdMAm6KZmQ+Ad4scKep6DgMHJzl/NfbgqFIpjNZmi1/ec9hhBCOqLDV1itWrUK3333HV5//XWIRCK8+uqr+Pvvv5GQkIBjx461ez9JSUlITU3F66+/DqPRiIKCAnzxxRe4+uqrAQDTp0/Htm3bAACzZ89GQ0MDPvzwQ1itVvz999/Iy8vr8AAPvc2pXQZ7QvfB7vL69vf421qpD25MvsaL1RBCehLGcTDt3YPSt95A0WMPQ5+znA9KsqhoBM+dhwGvvYnAy66goHQKm82GFSuWYOnSv/igNGTIcFx++bXQanXeLY4QQnqwDp9ZamxsRGRkJABAKBSC4ziIRCLceeeduP766/nritrjnXfewRNPPIGMjAyo1WpcddVVuOYa9wfjwsJCmM3uPtPBwcH46KOP8MILL+D9999HWFgY3nvvPURF9d2zDRabk5+Qdnp6332ep3tgzVMey89ntP9sJSGk73KZzTBsWAf9qhVwVFWdXCESwWf4COgyp0IeH0+DErSgvLwUK1YsQVOTAQCgUqmRlTUdERH9572FEELOVofDUkhICHbu3Ilhw4bBz88Pu3fvxrBhw6BWq1FdXd3hfX3yySctrjt48KDHcnp6Ov7444+Olttr1TaevP5rQJimjS37jry6g7A4T3aleWrMQggFvWt4SUJI57KVl0G/cgUMG9eD2Wx8u8hHA+2kydBNngKxzteLFfZsjY0N+OOPn/gBlQYOHIQJEzIhl8u9XBkhhPQOHQ5L11xzDebMmYMNGzYgKysLd911F6ZOnYr9+/cjMTGxK2rslworDPztyCB1G1v2DU7Oifd3f8YvT4uegiAldaEhpD9iHAfT7l3Qr8yBOX+/xzr5gAHQZWZDPWIUhBKJlyrsPbRaXyQnp+Hw4QOYODELAwcO8nZJhBDSq3Q4LM2dOxdhYWHQaDR48MEHYTabsXHjRkRHR2PhwoVdUWO/tOOQe6Q/pUyMIN/eNR792Xhrx0ceyxfGzfBSJYQQb3EZjWhctwb61SvhrK3l2wViMdSj0uGbmQ157AAvVtjzMcZQX18H/1Ou1xo7diKGD0+HWu3jxcoIIaR36nBYAoBp06YBAKRSKV544YVOLYi4lVS7J1E025x9vg/+mtKNKDScHBzk9YnPebEaQkh3s5UUo2FFDpo2bwRzOPh2kU4H3eRMaCdOhljTP7ojn4umJgNWrFiCmpoqXHnl9dBo3BO9SyQSSOgsHCGEnJUOhSWj0YhNmzZBJpNhzJgxzV58v/76a1x//fWdWmB/dWI+KB9l336DKzNW4IdDv/HL48PHQC6WebEiQkh3YE4njLt2QL8iB5bDhzzWKQYmQJeVDfXQ4RCIz+o7vX6FMYZDh/Kxdu1Kfv7DnTu3YtKkbC9XRgghvV+734UKCwtx4403orKyEgAQFRWFb775BsHBwairq8NDDz2EzZs3U1jqJI0m9xteSqy/lyvpWi9ueZO/PSwwFVcnXuLFagghXc1pMKBxzWo05q6Cs6GBbxdIJPAZPRa6zCzIo6K9WGHvYrVakJubg4KCwwDck8qOGJGOESPGeLkyQgjpG9odlt5++22kpqbiu+++g91ux7PPPot33nkH06ZNwyOPPAJ/f3/8+OOPXVlrv2G2Or1dQrdYUrTSY/nm1Ou8VAkhpKtZiwqhX5GDpq2bwZwnX+PE/v7QTc6CdsJEiNR9fzCbzlRcXIiVK5fBbDYBADQaLbKzZyAkJMzLlRFCSN/R7rC0a9cufP/99wgJCQEAPPnkkzj//PPxxx9/4MYbb8Rdd91FfaI7SXmdib+dEuvnxUq61j+Fy/jbT42hwUEI6WuY04mmbVugX5kD69GjHusUgwbDNysbqiHDIBDSFAEdtWHDGuzatY1fTkpKRUbGJEgkUi9WRQghfU+7w1JDQwMflAAgOtrdTeK///0vhg4d2umF9Wf7jtbxt/vqsOH5dYfAMY5fpmHCCek7nPoG6HPdXe1chpPTIAhkMmjGZkA3JQuy8HAvVtj7abU6AIBCocSUKdMQE0OjBBJCSFc4pytnRSIRBaUucKyyib8dFqDyYiVdw+5yYNHuT/nlmTF0ETIhvR1jDNaCI9CvzEHT9m2Ay8WvkwQGQZeZBU3GeIiUfe81rTswxjxGRk1KSoXVakVSUgoUCqUXKyOEkL6NhhnqgRqaTs5SLxT2vWHD7819zGP5/AHTvFQJIeRccQ47mrZshn5FDmzFxzzWKVNSocvMhiollbranQO9vgErVizG6NHjERERBeDkQA6EEEK6VrvDEmMMRUVFYIy12RYbG9u5FfZDxcfnWOqLZ5X+LVzusfzM2Ie9VAkh5Fw46urQmLsKjWty4TKePBsulMuhyZgA3ZQsSE/puk06jjGGvLw92LAhF06nEytWLMFVV10PmUzu7dIIIaTfaHdYstvtmDFjhkcbY4xvO9FFID8/v3Mr7Ge4U4JnqH/f61rxzylh6f4RtyNA0XcHsCCkr2GMwXLwAPQrc2DcuQM45fVKGhrm7mo3dhyEcoUXq+wbTCYjVq1ahuLiIgCAUChEaupQGsCBEEK6WbvD0tdff92VdZDj6hut/O2B4VovVtL58uoOeiwP0MZ4pxBCSIdxVgtK33wd1oIjJxsFAqiGDIUuMxvKwUke19SQs1dQcBi5ucthtbrfD/z8/JGdPQMBAUFerowQQvqfdoel9HTqG90dqvUW/naIf9/qhvf+7s/424+l3+fFSgghHVX5xWd8UBIqFNBOnATd5CxIAgO9XFnfYbPZsG7dKhw8uJ9vGzJkBEaPzoBYTJcYE0KIN9Crbw9TUWfmb/tr+06/9IP1RzyWw9R0LQMhvQFjDNX/+y+M291z+ojUPoh95T8QymRerqzvqa+v44OSSqVGVtZ0fkAHQggh3kFhqYcpKG/kbwfp+kZYcnEuvLPrY355RkyWF6shhHSEPmcZGlet4JejnnyGglIXCQ0Nw7Bho2A0NmHixEwayIEQQnoACks9TOUpZ5YkYpEXK+k8b+38yGN5OoUlQnqF+n//Ru2vP/PLUU8+A4kfDcrSWWpra9DQUIeBAwfxbWPGjKdrvwghpAehsNTD9MX3yKONRfztNyY9D7GQDjtCerrq776FfsXJ0StDb18AeVS0FyvqOziOw+7d27F58wYIBIC/fyD8/PwBgIISIYT0MGc1S6Ber8evv/6Kd999l28rKyvrtKL6s7JaEwAgLc7fy5V0jl8P//3/7N13eBRVGwXwsy2990pCLwEivbeEEkA6SBEQGyKiVBUU2yeogBRBRBGlWhBRepEQegcpIXQILY30XrbM98fChJgAWdjsbJLzex4eZ2ZnZ94sY5iz98694nILryawVHDYWyJzl7prZ2FQUijgO3EK7Bs3kbaoCiIjIx2bNq3DkSMHoNNpAQApKUkSV0VERI9i8Ff8Fy5cwMsvvwydTofc3Fy8/fbbuHPnDnr16oWffvoJTZrwH9RnUaDWAQDcKsDgDldSr2P3nf3ieu/qYRJWQ0SlkXH4EBLX/iau+44bD9ug+hJWVDEIgoDLly/iwIEIqNUFAAB3d0907twdzs7s2khEZK4MblmaPXs2+vfvj6NHj0Iu17/d398fEyZMwLx584xeYGWi1ujE5dx8rYSVGMc3Dz2r1MKrCZwsK9a8UUQVTea/p5Cwarl+RaFAwGczYdugobRFVQC5ubnYuXMLIiJ2QK0ugEwmQ9OmLdC//xAGJSIiM2dwy9LZs2exZMkSKBSKIn2rhw0bhgULFhiztkonr0AjLlf3dZCwkmd3NfWGuOxq5YyR9QZLWA0RPUnu9WuI+66wa7XH4KGw9PWVsKKKQafTYcOGtUhNTQEAODo6ITQ0DF5ePhJXRkREpWFwWLK2ti7xAdSsrCw+mPqMsnLV4rKdtUrCSp7dT+fXiMuv1h8uYSVE9DiCICBtdzgSf/9F3ObSqw+cQjpLWFXFIZfL0aRJC4SHb0e9eg3Rpk17qFR8dpOIqLwwOCzVr18fixcvxuTJk8VtmZmZmDlzJho3bmzU4iqb1Mx8cdm+HIelPE0eMtVZ4noVez8JqyGiRxEEAfdWr0T6/r3iNuewHnDr00+6oiqAjIx0ODgUdjuuWbMOnJyc4eHBybiJiMobg8PSlClTMHLkSKxfvx4FBQXo1asX7ty5Azs7Oyxbtqwsaqw0bsRmiMs+7nYSVvJs/r62VVzuW70HWxyJzJCg0+HunK+Qe/UKAECmUsF96Itwat9R2sLKMa1Wi1OnjuHUqWMIC+uFqlVrANAPB86gRERUPhkclmrVqoVt27Zh8+bNiI6OhpWVFapWrYrnn38etra2ZVFjpaHRFg7wYG9TPluWUvPScDD2mLgeWqW9hNUQUUm0Odm4/flnUCfe02+QyRD4+RdQublLW1g5lpqagt27t+PevQQAwKFD+xAQUE0cCImIiMong8PSwoUL0b9/f7z00ktlUU+ldi81V1yWl9PWmOmHvxCX2/u2hlzGGwUicyJoNIj9dqEYlFSeXvB/bxqUjhyt8mkIgoCoqLM4fHg/NBr9ID3e3r4IDQ1jUCIiqgAMDkvr16/HkiVL0LRpU/Tv3x9hYWGwtrYui9oqnfiUHKlLeCb3copOrDioVm+JKiGikmizshC7eKHY9U7l5YXAT2dApjT4nwICkJ2dhT17/sHt2zcB6AdzaN68NZ57rimDEhFRBWHwb/N9+/ZhxYoVCAwMxJdffok2bdpg2rRpOHHiRFnUV6m4OugnorVUKSSu5Okcjz8lLncLCGGrEpEZUScn48b7k8WgZFM3CAEffcag9JRu376J339fJQYlFxdXDBw4DI0bN2dQIiKqQJ7qX8kWLVqgRYsW+Pjjj7Fv3z5s374db775JlxcXPDPP/8Yu8ZK4/KdNACAp3P5bKnbfnO3uNy7epiElRDRA5q0NKQeP46kdWshqPXTE1jXqg2fce9AbmkpcXXll5WVNdTqAgBAcHATtGjRBkoGTyKiCueZfrOrVCrUqVMH0dHRuHPnDi5cuGCsuiolQRAAAM725e8G5r9d8IhIOpr0dGScOYW7p08hI+oCcP93CwDYNWkKnzfHSVhdxeDh4Ym2bTvByckZfn5VpC6HiIjKyFOFpfj4eGzfvh3btm3D+fPnUbduXfTu3RtLliwxdn2VUnkcavuzo7PF5ZfqDZGwEqLKSZORgax/TyLz5AnkXr5UJCABgKW/P6xr1oL74GESVVh+abUaHDt2CJaWVmjSpIW4vX79YAmrIiIiUzA4LA0ZMgTnzp2Dq6srevXqhZkzZ6JWrVplUVulk52nH0mpuq+DxJUY5nbG3SLrzTwbSVQJUeWizclG5onjyDxxvMSAZBMYANvGzWDbuCksvDjPz9NISkpEePh2pKQkQS6Xw98/EB4enlKXRUREJmJwWPLz88O4cePQunVrPsRqRFpd4RxLSkX5+Vx1gg7z/i1sUXwl6MVy2TJGVN4UJMTj5scfAlptke0Wvn6wb9oMTi1awCuoJlJTs6HR6B5xFHoUnU6Hs2dP4dixw9Dp9J+xr68/bGxsJK6MiIhMqVRh6datWwgICAAAvPXWW+K2klStWtVIpVUuCSmFcyzp/vPtsDmbdvBzqHX6h8YdLRzQxJPdUojKWvb5SMR++40YlGQWFnDp3hN2TZrB0scHAKBUlp8vXcxNRkY6IiJ2IDY2BgCgVCrRqlV71K8fzC+DiIgqmVKFpd69e+Ps2bMAgO7du5f4j4UgCJDJZLh48aJxK6wkYpOyxeVq3uWjG15ybgqy1IV1963RQ8JqiCqHuGU/IPPoEXHdpcfzcOs/UMKKKg5BEHD58gUcOLBHHOnO3d0TnTt3h7Ozi8TVERGRFEoVln766SdxedWqVWVWTGWWmpkvLnu52kpYSel9fOQrcTnEvx2aezWWsBqiik0QBNxbs6pIUPIc9Qoc27aXsKqKpaAgH0eO7IdaXQCZTIYmTVqgSZMWUCjK59x3RET07EoVlpo2bSouHz16FO+8806xfbKzszFv3jw0b97ceNVVIgpFYWudrZX5z9Xx7ZllRdb713heokqIKjZBEJBz8QKS/16PvOgbAAC5tTV8xo2HTe06EldXsVhaWqFjx644fHgfQkPD4OXlI3VJREQksVJ3atfpdCgoKMBPP/0EtVqNgoKCIn9u3bqFP/74oyxrrdAefgBbITfvPvG5mlxcTLkiro9uMJL9+InKQM7FC7j6+suImTdHDEoAUOXDjxmUjECtVuPChUhxjjsAqFq1OoYMeYlBiYiIABgwGt7SpUuxYMECyGQyNGzYsMR96tata7TCKpvrsRkA9CPhmXvw2Bq9S1y2V9kh2L2+hNUQVUx5N6MR8808cV1uYwvHDh3h0r0nFByR7ZklJMQhPHw70tPToFQqUatW4b9f7HZHREQPlDosjRkzBp06dcKAAQPw+eefF3vd2toarVu3NmpxlcmDbngarXkP8ZuSl4o9dw6K6x+1nCJhNUQVU87FC7g7/2tApwNkMji0agOP4SMht7CQurRyT6vV4tSpYzh16pjYonT37u0iYYmIiOgBgx6OqV27NhYtWoROnTqVVT2VVnaufkJadycriSt5vAX//iAue9q4w1bFb7iJjCnp7/VI2bpZXPcc9Soc27SVsKKKIzU1Bbt3b8e9ewkAAJXKAu3adULt2vUkroyIiMxVqcLSwoULxUEdTp8+jdOnTz9y30mTJhmnskomLlk/BLc5T/R7NvE8kvNSxPWPWrBVichYBEFA0rq1SP1nh7jNY8RLDEpGIAgCoqLO4vDh/dBo9F9MeXv7IjQ0DA4OjhJXR0RE5qxUYWnbtm1iWNqyZcsj95PJZAxLTykpPQ8A4OtmvsOG/3LxT3G5hVcTs3+2iqg8iV20ANnnzorrVaZ/AqtATvJtDIcP78fZs6cAAHK5Ai1atEZwcBOz/nKKiIjMQ6nC0o4dhd90RkRElFkxlZm9jQqZOWrcvZcldSklWn3xD2RrcsT1kfUGS1gNUcWScfyoGJQsvLzhO2ESVG7uEldVcdStWx/nz5+Bo6MzOnfuDjd+tkREVEpPNaHPwYMH0batvmtIVFQUNmzYgOrVq2PIkCFGLa6y0AkCMnPUAIA2DbwkrqY4tVaNo3EnxfX3mr4tYTVEFUve7VuIX/q9uO795jgGpWeUn58PpVIBhUL/T5yLiyt69RoADw8vKJXmP48dERGZD4P7IPzwww+YOnUqACAlJQWjRo3CpUuXsGzZMnz77bdGL7AyiE8ubLFxd7aWsJKSrb5YOH9WHeeaCHDwl7Aaooqj4N49xC5aIK57vf4GLH19pSuoAoiJuYO1a1fh+PEjRbb7+PgxKBERkcEMDkvr1q3DDz/oR0TbtGkT/P39sXr1aixbtgybNm0yeoGVQUJqYVjydDa/0eVOJ0aKy28996qElRBVHJr0dNye8Rk0qakAAM9Rr8ChRSuJqyq/NBoNDh3ah40b1yErKxOnT59AamrKk99IRET0GAZ/zZacnIygoCAAwOHDhxEWFgYACAwMRGJionGrqyTSMvPFZQcb85pHJS0/HTpBP/eTh7Ub5DI+EE30rNTJyYhZtAC6HP0omE5dusGxbXuJqyq/kpISER6+HSkpSQAAS0srdOjQGc7OLhJXRkRE5Z3BYcne3h4pKSmwsLDAiRMnxFHyHmwjw524dE9cdnGwlLCS4m5l3BWXOwd0kLASoooh/85t3J33NbSZGQAAm6D68Bg8VOKqyiedToezZ0/h2LHD0Om0AAB//wCEhHSDra2dxNUREVFFYHBY6ty5M15++WXI5XIEBASgfv36yM/Px8yZM9GiRYuyqLHCuz+JPACY3XDcF1Iui8u1nWtKWAlR+Ze8ZROSN/wlrrv06gPXXn0krKj8ys7Owq5dWxEbGwMAUCqVaN26PYKCgs3u9ygREZVfBoelqVOnYsWKFcjMzMSLL74IQP/tXmpqKr766iujF1gZxCTpu+JYqMyvi9vN9NvisquVs4SVEJVv2efPFQlKjp1C4dann4QVlW9KpRLp6ekAAA8PT4SGdme3OyIiMjqDw5KFhQVGjx5dZJu1tTV+/vlnoxVV2WTl6ocND/C0l7iSonLUObibFQsAcLCw57e1RE8p4/AhxC9fJq57DB8Jp44hElZU/llaWiE0NAyxsXfRpEkLKBQKqUsiIqIKyOCwpNPp8P333+Pvv/9GTIy++0NAQACGDBmCl156yegFVgYKuQxanYD8Aq3UpRTxzeml4rKXjYeElRCVX5n/nkL8zz8CAGQWFvB5cxxsGzSUuKry5+bNG7hx4yo6deoqfnHj51cFfn5VJK6MiIgqMoPD0qJFi/DLL7+gX79+qFGjBnQ6Ha5cuYKFCxfC0tKSE9M+Ba1O/9BS3UDz6eaWp8kXW5UA4JX6L0pYDVH5o8nIQPKGv5B+YJ+4zXvMWwxKBlKrC3Do0H5cuHAOAODp6Y2gIH6GRERkGgaHpY0bN2LJkiVo0qRJke1dunTBzJkzGZYMpHtodAeNVnjMnqY1ef9H4nK/Gj1hb8GRpYhKQ9BqEb/iJ2QePwZoC1uLPUaOgl3DYAkrK3/i42Oxe/cOpKenAQCsrW1gZ2de3ZWJiKhie6p5lho1alRse/PmzcVueVR62ocCko+brYSVFMoqyC6y3smvrUSVEJUvgiAg7ofvkPXvKXGbTd0geIx4CRYe7MpaWlqtFqdOHcWpU8ch3P9CqVq1GujQoQusra0lro6IiCoTg8OSj48Pzp8/j4YNi3aDiIqKgpubm9EKqyx0usKwpJCbxwAK/947Jy4PrtUXCjkfnCZ6EkEQkPTXn0WCUsCnn8PSz1/Cqsqf1NQUhIdvR2JiAgBApbJAu3YhqF27LgeZISIikzM4LPXu3Rtjx47FyJEjUbOmft6dy5cvY/Xq1ejfv7/RC6zotGYYljZc3yout/NtJWElROVH/M8/IvPIYQCAys0dVT7+DAobG4mrKn8OHtwjBiUfH1+EhITBwcFR4qqIiKiyMjgsjR49GlqtFj///DPS0tIAAPb29hg8eDDGjx9v7PoqvAJN4TMNcjMIS2fuRSJfWwAAcLVy4Te5RE8g6HRIXPubGJQAwH3YiwxKT6lDh874889f0ahRMwQHN4Zcbn7zzxERUeVhcFhSKBQYN24cxo0bh8zMTOTn58PV1ZU31U8p+/4cSwBgZ62SsBK9f27tFZcbe3DEKaLH0ebm4tYnH0KTkgIAkFtZIeCTz6Fyd5e4svIjOvo6/PyqQKXS//5zcHDEiBGvQqWykLgyIiIiA8PS3bt3cfDgQahUKnTq1AkuLi6wt+fIRM8iNStfXLa2MDi7Gt2tzDsAAAu5Cn1r9JC4GiLzlXf7FuKX/SAGJYW9A6p88BGDUinl5+fhwIE9uHLlIurXD0b79qHiawxKRERkLkp9dx4ZGYnhw4dDEATodDrMnTsXq1evRvXq1cuyvgovLjlHXLa3lbZladWFteJy5yodJKyEyLzlXr2CO7O+ENcdWrWBx/CRkFtaSlhV+RETcxu7d+9EVlYmAODWrWgUFOTDwoKfHxERmZdSdwZftGgR+vXrh3///Rf//vsvOnXqhG+++aYsa6sUMnMKu+G5OlhJVodap8Gx+MJRvGo5MwQTlSTvZnSRoOQc1gOer7zGoFQKGo0Ghw7txcaNf4pBqXbtenjhhREMSkREZJZK3bL077//4quvvoJSqX/LhAkTMHDgwDIrrLJ4MIeIlYUCSoV0DzJHJV0ssl6TYYmoGF1eHhJWrxTX3YcMg3PnrhJWVH4kJd1DePh2pKQkAwCsrKzQoUNnVK9eS+LKiIiIHq3UYamgoAAuLi7iuru7uzgaHj09jVYHALCxkvZ5pR/PrxaXZ7T+QMJKiMyTJiMDd+fNQcFd/XN9jp1CGZRK6fbtm9i2bQN0Ov3vuypVAtGpU1fY2tpJXBkREdHjlfoOnaPdlY3oOH1XFKWEw+NGJl0osu5s5SRNIURm6t6va5AWES6uW9epC4+hL0pYUfni7e0Le3sHZGdnoXXrDggKash/U4iIqFyQfvi1Si45PRcAkJOvkeT8giDg+3MrxPX3m74jSR1E5kinViPx91+Rvm+PuE3h6AifMW9Bxvl/HkkQBKjValhY6Ee1U6lU6Nr1eahUKjg5OUtcHRERUekZ1A1vyJAhT9z2+++/G6eySiInXz8pbdZD8y2Z0rmkqCLrVRz8JKmDyNwUJN5D3PffIf/WTQCA0tkFXq+/Aeuatdgq8hi5uTnYuzcc+fm56N17kDiprLu7h8SVERERGa7UYalPnz7FbhCqVq1q9IIqGysLBXLzNajl7yTJ+ZdGrhKXX60/XJIaiMxNxrEjuLdmFXS5+pZfm/oN4P3qaCg4r9xj3bx5A3v2/IPcXP2UCBcuRKJ+/WCJqyIiInp6pQ5LX331VVnWUWkpFfoAamWhkLgSoLFHQ6lLIJKUIAhI2bYFyX+v12+Qy+HWtz+cw3qw291jqNUFOHRoHy5ciBS31a8fjNq160lYFRER0bPjM0sS02j1Q4f7uNqa/Nw7bu4Wl5+v2s3k5ycyJzmXLyHp7/XIu3ZV3OY7fhJsg+pLWJX5i4+PRXj4dmRkpAMAbGxs0alTVwQEsOcBERGVfwxLEkvNzAcAKBSmfQZCJ+iw+cZOcb2eK+c6ocorZftWJK1fJ65bePvA550JsOBzNo+k0+lw4sQR/PvvcXG+uGrVaqJDh86wtraWuDoiIiLjYFiSmEIug1YnID27wKTnXXmh6EAcAQ7+Jj0/kTkQdDokrFqOjIMHAAAyCwu49Hgezp27Qm5lJXF15k0mk+HevXgIggALCwu0axeCWrXqcvALIiKqUBiWJCQIAnQ6/Teyns6m+yZWEAScTDgjrs9s86HJzk1kLnT5+bgz5yvk34wGAChdXOE7fhIsfX0lrqx8kMlkCAnphv37I9C2bUfY2ztIXRIREZHRPfUTy2q1Gnfu3DFmLZWOVidAuL9sY2m63LrtZuHkmnYqWzhZOprs3ETmQNBocPvLGWJQAgD/96YyKD1GVlYmdu/egfz8fHGbra0dunfvzaBEREQVlsFhKS8vD++//z4aNWqE7t27AwAyMjLw2muvISMjw+gFVmQ5eYUT0eYWaE123m3Ru8Tl95txElqqXNQpKbj56XQU3NV/2WNVtRpqfPs9VG7uEldmvq5evYy1a1fh8uULOHhwz5PfQEREVEEYHJbmzJmDixcv4uuvv4ZCUTjctVarxddff23U4io6tUYnLpuqG15yboq47GrlAhcrZ5Ocl0gqOnUB1KmpyL9zB1lnTuPWp9Ohjo8XX/eb/C6fT3qEvLw87Nq1Dbt2bRVblKytbcQBHYiIiCo6g/t+7dy5E2vWrEFgYCDef/99AICDgwO+/PJL9O3bF//73/+MXmRFlZ2nFpflJnooeu/dQ+Jy72ocLpwqLnVyEu7OmwN1QkKJr9sE1YfPuPGQq1Qmrqx8uHv3Nnbv3oHs7CwAgL29A0JCusHXl4PBEBFR5WFwWMrOzkZgYGCx7S4uLsjJyTFGTZXGwy1LNlameWYp4s4Bcbm+GyeMpIpJm5uL2MWLHh2U6jeE7zsTONFsCTQaDY4dO4izZ/8Vt9WuXQ/t2nWChYWlhJURERGZnsF36FWqVMGxY8fQokWLIl0xduzYAR8fH6MWV9GlZOSJy9YmGODhUsrVIutWSt74UMWjzcrC7S8+h/qePihZ16kLp46doLC1g9zWFkoHByid2P30USIjT4tBycrKCh06dEb16pyHjYiIKieD79CHDRuGt99+GwMGDIBOp8Py5ctx/vx57Ny5Ex9+yCGoDZGUXhiW7KzLvitQ+O194vIbDV4q8/MRmZqg0SDuhyViULJr0hTeb4xlC5IBGjZshMuXL8LOzh6dOnWBra2d1CURERFJxuCwNHjwYCiVSqxZswYKhQLff/89qlatiq+//hphYWFlUWOFpVIW3sDZ21iU6bnUWjUuplwR1xu6B5Xp+YhMLftCFO6tWVU0KI15i5OkPkFGRjoEQQdHR31rm0KhRJ8+g2BlZcXPjoiIKr2n6vs1YMAADBgwwNi1VDoabWE3RqWibG9KDsQcEZerOQaU6bmITC1pw19I2bJJXLepFwTv18fwZv8xBEHApUtROHhwD5ydXdCv3xBxhFNra9NNkk1ERGTODA5LGzZseOzrffv2fcpSKp+c+6PhKRXyMr+pW39ti7g8qfHYMj0XkSmlhu8qEpQsvLzhM/ZtyJSmm+i5vMnJycG+fbsQHX0dAJCYeA/x8bEc6Y6IiOg/DL6bmDp1askHUiphZWXFsGSAxLRcAIC8jL/8/u/ADvy2nSqKvJs3kfj7LwAAhYMD/Ca/B0tfP4mrMm83b17Hnj27kJurH73UyckZoaHd4enpJXFlRERE5sfgsHTu3Lki61qtFjdu3MDSpUsxcuRIoxVWGaRk6Cd5LHhoCPGy8Pe1reLyuOdeK9NzEZlK5qkTiF+2VFz3HPESg9JjqNUFOHRoHy5ciBS31a8fjFat2kPFuaaIiIhKZPAQURYWFkX+WFtbIygoCB999JHBE9LGxMRg9OjRaNGiBTp16oQ5c+ZAp3t8cEhISECjRo2waNEiQ0s3O452+kEdlIqyG6nraup13M2KBQA4WzqhrguHAKbyL23/XsQtWQxBrYZMqYTnSy/DrlETqcsyW7m5uVi7drUYlGxsbPH88/3Qvn0ogxIREdFjGK1Tv4ODA27dumXQe95++20EBQUhPDwcycnJeOONN+Dm5oaXX375ke+ZMWOG+BByeafT6Qd4sLcpu5uVBad/EJeH1eGgHFT+pR/Yh3u/rBbXfcaNh239BhJWZP6srKzg7u6BjIx0VK9eEx06dIaVFQdxICIiehKDw9LBgweLbcvLy8O2bdvg5VX6Pu+RkZG4dOkSli9fDnt7e9jb22PUqFFYuXLlI8PSvn37cO3aNXTs2NHQss1S8v1JaT2dy+am5UrqNXE5yLUO6rnWLpPzEJlKyrYtSPrrTwCAzMICniNHMSg9glarFZdlMhk6dOiMatVqokaN2nxukYiIqJQMDkuvvfYaZDIZBEEost3JyQlfffVVqY8TFRUFX19fODo6ituCgoIQHR2NrKws2NkVnQgxLy8P//vf/zBz5swnjsj3KHK5DPKyHk2hFBT3u92lZRUA0NelVBq/K97Je2fE5f61epTJOcg0HlwzijLssmnu0o8cFoOSwtYWVSZPgXW16hJXZX4EQcC5c6dx+vQpjB79unjN2NnZom7dehJXR+aMv2fIULxmyBDl9XoxOCzt3r272DYrKyu4uLgY9G1lWloaHBwcimx7EJxSU1OLhaXFixfjueeeQ8uWLZ86LLm42JrVN6ruTtZITs9DUnoenJ1tjX78yKQL4nJ9/xpm9bPT03FwqJxdp3Lu3EXcyhXieoMvZ8A2oIp0BZmpjIwMbNy4ETdu3AAAbN++HQMHDpS4KipvKuvvGXp6vGbIEOXtejE4LK1YsQIffvihUU7+39apR7l27RrWrVuHzZs3P9P5UlKyzaZlycHBGpdupQIAavo5ITU126jnuJeThIz8LABAe79WSEvLMerxybQeXDMZGbnQast29ERzk3fnDm58dP93jkKBqh9OR4GDKwqM/P9MeXflyiXs2bML+fn6UTbd3NzRvn37SnnN0NOpzL9n6OnwmiFDmOP1UprGCoPD0vbt2zFu3Lgi3eeehouLC9LS0opsS0tLg0wmg4uLi7hNEAR8+umnePvtt+Hu7v5M59TpBHFQBXPgbG+J1Mx8pGXlQ2Pk4cOXnF4hLtdyqmH045M0tFpdpfq7zL1xA3e+KBxl033AIKiqVK1Un8GT5OXl4cCBCFy9eknc1qhRU7Rq1Rbu7o5ITc3m50UGqWy/Z+jZ8ZohQ5S368XgsPTee+9h2rRpGDBgAPz9/YsNO1u1atVSHad+/fqIi4tDSkqKGI4iIyNRo0YN2NoWprzY2FicOHECV69excKFCwHoZ5+Xy+WIiIjA33//beiPYDZSM/XfAHu72hj1uIIgIDY7Xlx/zr2+UY9PZArq1FTEfrdQXLdr2hxOXbpJWJH5uXv3Nnbv3oHsbH0rsr29A0JDw+Dj48dnFImIiIzgqcISAERERBR5BkYQBMhkMly8eLFUx6lXrx4aNGiAuXPnYtq0aUhISMDy5cvxyiuvAADCwsIwY8YMNGrUCPv27Svy3i+//BJeXl547bXyO8Gq/vMCBAEoUGuf/AYDhN8u/Lyec6/PZ5Wo3NFmZyNm3hxo77c+q9w94DNmrLRFmaG4uBgxKNWpE4S2bTvCwsJS4qqIiIgqDoPD0qpVq4x28oULF+Kjjz5CmzZtYGdnhyFDhmDYsGEAgOjoaOTk5EChUBQbktza2hp2dnbP3C1PShqtgAePbFXxtDfqsQ/EHBWXB9fuZ9RjE5lC3PeLURB3fzLlLt3gPnioxBWZpyZNWuDevQTUqROE6tVrSl0OERFRhVPqsBQcHIyzZ8+iefPmRju5l5cXfvzxxxJfu3z58iPfZ8gQ5eYqO1ctLmu1xn2OKjkvBQDgZu0KBwvjBjGispb45x/IuagfydG2UWO4vTBE4orMg06nw+nTJxEQEAg3Nw8AgFwuR8+efaUtjIiIqAIrdaf20o5cR6WTkZ0vLrs4GK/bTIG2MIT52JZ+kmAic5B15jRSd2wDAMgsreD18mvsRgogPT0NGzb8gWPHDiI8fDs0Go3UJREREVUKpW5Z4g2LceUVFD6npDLig9hrrxQOeBHkWttoxyUqa9lR5xH77Tfiut+ESVDYGHfwk/JGEARcvHgehw7thVqt/yJEpbJAfn4elEq7J7ybiIiInlWpw5JWq8Uff/zx2BYmmUyGF154wSiFVXQPd72zt7Ew2nGvpl4Xl1t4NTHacYnKUsaxo4j/8Xtx3fOll2Fds5aEFUkvJycHe/fuws2b+v+n5XI5mjZticaNm0Mu50h3REREplDqsKTRaPDxxx8/dh+GpdLT6ArHl1cYaaJcnaBDcp5+oltXK2eoFKonvINIeml7duPeL6vFdZcez8OxXQcJK5JedPR17N37D3JzcwEATk7O6Ny5Ozw82LWWiIjIlEodliwtLXH27NmyrKVSScssfGZJoTDOt8TnEqPE5eZsVaJyIOPY0SJBybVvf7g+31vCiqSXmHgP27dvFNcbNHgOLVu2KzanHREREZU9g4cOJ+NQPhSQlEZqWVp98Q9xuY2P8UYtJCoLqf/sROIfv4nrni+/Bsc2bSWsyDy4u3ugTp0g3L59EyEh3VClSqDUJREREVVapQ5LHA3PuHQPfZ7GGOChQFuAPG1ha5WzldMzH5OorKQf2FckKHm9/gYcWrSSsCLpaLVapKYmi8OBA0Dbth2h0+lgZWUtYWVERERU6rDUp0+fsqyj0tHpCsOS3AgtS3ez4sTlwbX6PvPxiMpK1pnTSFi9Ulz3nfQubOsFSViRdFJSkhEevh2ZmekYPHgk7Oz086JZWBhvOgEiIiJ6eqUOS59//nlZ1lHpFAlLRhiW/ddLf4rLNZyqPfPxiIxNm5WF5C0bkb53D6DTQaZUwmfcO5UyKAmCgMjI0zhy5AC0Wv00ApGRZ9CqVTuJKyMiIqKH8ZkliTzcDe9ZW5ay1NmIy04Q1z1t3J/peETGlh8bg9sz/wchv7CrqO/4SbCpW0/CqqSRlZWJiIiduHv3NgBAoVCgZcu2aNiwscSVERER0X8xLEmkaMvSsx3rYMxRcdnXzhsKueLZDkhkRJnHjyF+5XIxKCmdneE+eFilDEpXr17C/v27kX//s3B1dUfnzt3h6uomcWVERERUEoYliWi0hfMsyZ6xG15U8mVx+f2m7zzTsYiMKePYEcT/+IO47tS5KzyGDJOwIulEROzEpUuFw/s3atQMzZu3gkLBX8NERETmiv9KS+Reaq64/KyT0t5IvwkAsFRYsFWJzEbmyRNFgpJzl25wHzxUwoqk9aD1yN7eAaGhYfDx8ZO4IiIiInoShiWJ2FkXTjBpoXr6gJOrySs8psr2mWoiMpb0QweQsPwncd3rtdFwaNlawopMTxCEIq3GDRs2hk6nQ1BQQ452R0REVE48+wQ/9FSu300HAFhaPFtL0KHYY+Jy3xo9n+lYRM9KEAQk/LJaDEoyS0v4vDOh0gWle/cSsHbtKsTFxYrbZDIZGjVqxqBERERUjjAsScTZQX/DlF+gfabj/H1tq7hck0OGk4R06gIkrPwZ6Xt2AwDk1tbwmzAZdg2fk7YwE9LpdDh58hj++us3pKQkY/fu7VCr1VKXRURERE+J3fAkEpeUDQCwtnz6vwK1tuhNmL2F3TPVRFQagk6HnKjzKIiPgyYtTf8nPQ25ly6K+8itrRHw6edQVaJR3tLT0xAevh0JCfoJopVKJRo1agalkr9miYiIyiv+Ky6RvAINAMDO+un/Ck4mnBGXB9bs/awlET2RLj8ft2f+DwWxMY/cR2Fvj4BPPofSycl0hUlIEARcvHgeBw/uhUaj/wLD09MbnTuHwdHRWeLqiIiI6FkwLEnE3sYCAJCYlveEPR/tUupVcbmtT4tnronocXKvX8Odr2YCD02orHBwgNLJGUonJyidnKFy94BT5y6Qq1SPOVLFkZOTjb17d+HmzRsAALlcjqZNW6Jx4+aQy9nLmYiIqLxjWJKI9v6ktLX8HJ/6GNHptwAA9VxqQ6WoHDenJA1tZiZiF30jBiWrqtXgO34SFHaVu+tncnKSGJScnJzRuXN3eHh4SVwVERERGQvDkkR098OS/BnmWErPzwAAVHHgfC1UdnIuXUTs4oXQ5ernBrNv0Qper41+5smUKwJ//wA0aNAIgICWLdtBVUla1IiIiCoLhiWJaJ8xLOVp8qAR9CPpcWAHKisZRw4hfsXPgFZ/rTm0agOvV1+XuCrpxMXFIDs7CzVq1Ba3tW3bkcGRiIiogmJYkohGowPw9GHpfFLhyGOuVnyInIxLEAQkrV+H1B3bAAByKyt4jnwZdk2bSVyZNLRaLU6cOILTp09AoVDC3d0Tjo5OAMCgREREVIExLEkk9/5oeMqnfAj8WvpNcTnQoYoxSiICABQkJCBm0Xyo4+MBAHIbW/i/OxWW/v4SVyaNlJQkhIdvR1JSIgD9IA7p6WliWCIiIqKKi2FJImq1vmXJ2lLxVO8/EHNEXGY3PDKW7AtRiF28CEK+fpRGhaMj/Kd+CAt3D4krMz1BEHDu3GkcPXoA2vvdEH19/RESEgZ7e3uJqyMiIiJTYFiSiFqrD0sqpeFhKbMgy9jlECE78hxiFy+EoNG3elpVqw7vN8dB5Vz5unlmZmYiImIHYmLuAAAUCgVatmyHhg0bsdsdERFRJcKwJJE7CZkAAKXC8BuvC8mXxeVhdQYYrSaqvNRJiYj5Zp647jFiFJw6dJSuIAlptRr89ddvyM7Wfynh5uaOzp27w8XFTeLKiIiIyNQ4a6JErCz0LUo5+RqD33v9oeeVmno2MlZJVEnl3bqJ2zP/J667DxlWaYMSACgUSjRr1hIA0KhRMwwYMIxBiYiIqJJiy5JEVEo58gq0sHiKbniHYo+Jy5YKC2OWRZVMQUIC7s6bA112NgDAbcAgOHfuKnFVppeWlgonp8LuhnXrNoCnpzdcXd0lrIqIiIikxpYliWi0+nmWXBwsDXrf6XuR4nILryZGrYkql4KEBNydO1sMSu4vDIVL954SV2VaGo0aBw7swa+/LsedO7fE7TKZjEGJiIiI2LIkldz73e8UBs6ztPNWhLjcNaCTUWuiyiM/JgZ3v/4K2kz9s3PuQ16Ec+cuEldlWvfuJSA8fDvS0lIAAEeO7Ief33AO4EBEREQihiUJaHU6cTlfrXvMnsXdyYwRl71sK99wzvTs1CnJiPlmnhiUXPv2r1RBSafT4d9/T+DkySPQ3f9/MSCgGjp16sKgREREREUwLEmg4KGAZG+jKvX78jT54nIr72ZGrYkqh7zbtxCzcD60aWkAAI9hw+EU0lnaokwoPT0N4eHbkZAQBwBQKlVo27YD6tZtwKBERERExTAsSSA7r3AEPAtl6R8bOxJ3QlwOcPAzak1U8WUeP4a4pUvEdde+/StVULp27TIiIv6BRqMGAHh6eqNz5zA4Ola+eaSIiIiodBiWJKDVFrYs2VqVvmXpz6ubxOX6rnWNWhNVbIl//oHUHdvEddc+/eDSs5eEFZmera0dtFoN5HI5mjZthcaNm0Eu5xg3RERE9GgMSxIQHlq2UJXuZq1AW1Bk3dnKyXgFUYUlCAKS/vgdqbt2itu83xgL+2bNJazKdARBELvXeXv7om3bjvD09IaHh5fElREREVF5wLAkAZ2uMC6V9jmJu1lx4vLzVSvfPDhkOEGrRdLf68WgJLe2hu+EybCuXkPiyspeQUEBDh7cA1dXNwQHFw6x36ABJ3EmIiKi0mNYkoAgFIYleSnD0txTi8XlOi41jV4TVSyCTofbX85A/s1ocZv/e9Ng6V9FwqpMIy4uBrt370BGRjoUCgX8/ALg6uomdVlERERUDjEsSeChrITSZCWdUHR48aqOAUauiCoSnboA8T/+IAYlmVKJKh//D5Y+PhJXVra0Wi2OHz+M06cLB0KpWrUGbG1tJayKiIiIyjOGJQnoDGxZOhb/r7jc0a9NmdREFYMmIwN3Zn0BdUI8AEDl5YWA6Z9CbmUlcWVlKyUlCeHh25GUlAgAsLCwRIcOoahZs47ElREREVF5xrAkgYfDUmlalqLTb4rLvap1K4OKqCLQ5eUVCUpW1WvAb+LkCh2UBEHAuXP/4ujRg9BqtQAAX19/hISEwd7eXuLqiIiIqLxjWJKA7qFedXL5k9PS9bSb4rKVsuLe+NLTEwQBd+Z8JQYlh9Zt4DnqVcgq+NDYubk5OHnyKLRaLRQKBVq2bIeGDRtxglkiIiIyCoYlCeTkqcXlh0fGe5T4nHsAACsFgxIVlx8Tg5gFc6FJTQEAWFWrBs+XX6sUgcHGxhbt24fi9OkT6Ny5O1xcOJADERERGQ/DkgQefk7pSZPSxjw0ZLg1W5XoPzJPHEfcD98V2ebz1vgKG5Ty8nJx/fpVBAU1FLfVrFkH1avX4gSzREREZHQMSxLQPNSapFA8/qb2QvJlcXlU0NAyq4nKn6QNfyFly6Yi2wJnzoLS0VGiisrWnTu3EBGxA9nZ2bC2tka1aoVD6DMoERERUVlgWJKAWqMVlxWKx9/kHY07KS5X45DhBP1kszGLvkHO+XPiNp9x42H3XMWccFWtVuPo0QOIjDwjbouPjysSloiIiIjKAsOSBFIz88Vl5RMGeHjwvBIAyGX89ryy02RlIua775BzIQoAILO0QpUPpsPS10/iysrGvXsJCA/fjrQ0/fNY1tbW6NixK6pWrS5xZURERFQZMCxJwEKpEJetLB7/VyCXyaETdKjuWLWsyyIzlx19Ezdnz0VBbCwAQGFnD+833qyQQUmn0+Hff4/j5Mmj0N0fPjIwsBo6duwCGxtOMktERESmwbAkgczcAnFZpXx0a9GtjDvQCfobxepOgWVdFpmx9COHEfPD9+K6XZOm8H59DGTKivm/8N69u3Dpkr71TKlUoW3bjqhbt36FHbiCiIiIzFPFvNMyc1k5hUOHKx8zwMOm6zvE5VrO7HZUGamTEpG2dw9Sd+0UtzmH9YDbgEEVOjg0aNAIV65chLu7Jzp37g5HRyepSyIiIqJKiGFJAjZWhR/74254b6TfBACo5CrUdalV1mWRGREEAQkrf0bGwQNFtvu9/Q5sghtLVFXZycnJgZWVlTiqnbu7B/r0eQGenl4c6Y6IiIgkw7AkAeH+yOGWKsUj99HoNCjQ6VugqjsGmqAqMhea9HTcnTcHBTF3xW0Wnp6o+tJwKOoFQ6PRSVid8d24cRV79+5Cw4aN0bRpS3G7t7ePhFURERERMSxJQnc/LT3uC/Mb6bfE5drONcq6JDITeTdv4vaMTws3KBTwn/I+7OvWgbOzLVJTsyWrzdgKCvJx8OBe8dmkkyePok6dINjZ2UtcGREREZEew5IEHrQsPa4L3pnE8+Lycx4NyrokMgOp4f8gaf06cd2+WXN4vvI65CqVhFWVjdjYu9i9ewcyMzMAALa2dggJ6cagRERERGaFYUkCwv209LiwdDnlqrjsYeNW5jWRdARBQMrmjUjetEHc5twtDG4DB1e4QRy0Wi2OHz+M06dPiNtq1qyNdu1CYWVlJWFlRERERMUxLElAd79l6XHz0SbkJAIA7FScU6YiEwQB8cuWIvPYEQCAwt4BvuMnwSowUNrCykB6eip27NiC5GT9tW1paYn27UNRs2YdiSsjIiIiKhnDkgSe1LKk1mkgQL8PB3eouNRJiYj/6UfkXr0CALDw8obv5PegcnaWuLKyYWFhiZwc/TNXfn5V2O2OiIiIzB7DkgQKn1kq+fVbGXfE5XqutU1QEZlSzuVLSFz7G/JvFw7iofLygt9706B0cJCwsrJlbW2DkJCuSE9PQ4MGjSpcF0MiIiKqeBiWJPCgZUn+iJvFE/H/iss1naqZpCYyjfSDB5Cw4qci26xr1oL3m+MqVFASBAFXrlxEbOxddOzYRQxGAQG8nomIiKj8YFiSwI1Y/Qhgj/pe/XLqNXHZ09bDBBWRKaiTEpGw8mdx3bV3Xzi0bgOVm7uEVRlfXl4u9u3bjevX9d0LfXz8ULt2PYmrIiIiIjIcw5IEvN1scfFWKlIy80t8PTE3Wb+fracpy6IyoisoQMrWzUjduV3sg+n+whA4dw2TuDLju337JiIidorPJjk4OMLBwUnaooiIiIieEsOSBIT7w+F5u9oUey1Xkysue9qwVam8Uycl4s7sr6BJSRa3ufR4Hk5duklYlfGp1WocPXoAkZFnxG316jVA69YdYGFhIV1hRERERM+AYUkC2vthSamQF3stsyBLXK7vVtdkNZHx5Vy6iNjvF0OXpf87tfDzh+eIl2BdvYbElRnXvXvxCA/fjrS0VAD6gRw6deqCwMDqEldGRERE9GwYliTwICzJS5ho6VzSBXHZzcrFZDWRcWnS03H361niulNIKNyHvAiZvHhALs8EQcCBA3vEoBQYWB0dO3aBjU3xVlMiIiKi8oZhSQJ5BRoAJY+Gl5B9T1z2t/c1WU1kPAUJ8YhdvEhc9xg+Ek4dQySsqOzIZDKEhHTDhg1/oGXLtqhTJ4hDghMREVGFwbAkgTsJ+m5ZJd1TyuUKcdlKaWmqkshIcq5cRsyCuRAKCgAA9i1aVaigJAgCrl69hOrVa0Kh0P/6cHZ2wYgRr0KpVElcHREREZFxMSxJwNvNBvfScnErPrPYa1qdFgDgY+tl6rLoGWWdO4O4H5aIQcmlVx+49u4rbVFGlJOTjYiIf3D7djSSkhLRunV78TUGJSIiIqqIGJYkoNXqn1mqX8212Gux2fEAAKWcfzXlhaDVImHlz8g4fAgAIFMq4TX6Tdg3biJxZcZz/fpV7Nu3C3l5eQCA27ej0bx5ayiVvE6JiIio4uKdjgRu39N3w1MqivfDu5VxR/8aw1K5oElLRdzS75F75bK4zev1NypMUCooyMfBg3tx6VKUuK1hw8Zo2bINgxIRERFVeLzbkYDi/ih4ao2u2GtOlo5Iy0/H7cy7pi6LDJRz5TLivvsW2ix9d0rLgED4vPUOVC4VYxTD2Ni72L17BzIzMwAAtrZ2CA0Ng59fFYkrIyIiIjINhiUJ2FmrkJqZj6xcdbHX0vLTAQAh/u1MXRYZIP3gfiSsXglo9c+YOYWEwm3AC5BbVoxBOa5du4J//tkirtesWQft2oXAyspKwqqIiIiITIthSQK6+/MsBXjaF9mu0WnE5XxtvklrotLR5eUidsli5ESdF7d5jnwZju07SFiV8fn7B8DOzh5qdQHatw9FzZp1pC6JiIiIyOQYliTwqElpk/NSxeUAe3+T1kRPplOriwUl7zFjYd+0uYRVGYcgCFCrC2BhoW8Zs7S0RFhYb9jY2MDOzv4J7yYiIiKqmBiWJJCSoR9RTPGfsHQusfAhei9bD5PWRI+ny8/HndlfIv/WTXFb4MxZsPD0lK4oI8nMzMDu3TugVCrRs2c/cVJZD4/y/7MRERERPQuGJQkUPBjY4T+D4Z1NLGyx8LXzNmFF9Di5168hfvkyqOPjxW0B//ui3AclQRBw5cpFHDgQgYL7c0NdvXoJtWrVlbgyIiIiIvPAsCQBGyslcvI0SM0o+lxSdMZtcZlDh0tP0GqRsGoFMg4dELfZ1G8AnzFjIbeylrCyZ5eXl4t9+8Jx/fpVAIBMJkPjxs1RvXotiSsjIiIiMh+8I5fAgwEeqnjaidtS89LE5RpOVU1dEv2HJjMDt//3CTSphc+RuQ14Ac5h3cVuauXV7dvRiIj4Bzk52QAABwdHhIZ2h7e3j8SVEREREZkXhiUJPJhfSaGQi9tS7w8ZDgCtvcv/gAHlWXbkOdz7dY0YlJTOLvAcOQq2DRpKXNmz0Wo1OHRoH86fPytuq1evAdq06QCVykLCyoiIiIjME8OSiQmCII6G9/AAD0m5yeKyj52XyesiQJOejpTtW5AWvkvcZteoCbzHjIVMoZCwMuOQyeRISUkCAFhb26BTpy4IDKwucVVERERE5othycQ0WkFcTssqfGYpMSdJXHaxcjZpTQTk37mNW599XLhBoYD7oMFwCu1S7rvdPSCXyxEa2h1Hjx5EmzYdYWNjI3VJRERERGaNYcnEBKEwLHm5FN6sxmYniMu2Kt7EmlLezWjcmf2luK50doHP2+NhVSVAwqqeXVpaKk6cOIyOHbtCpVIBAOztHdClSw+JKyMiIiIqHxiWTEwoslbYYqHRaUxdCgHIOHYU8T//CGi1AADXfgPg2rOXxFU9G0EQEBV1DocP74NGo4GFhSU6dOgsdVlERERE5Q7Dkqk9lJYe7t11PvkiAKChW5CJC6q8Mo4fRfyP34vrzt26l/uglJ2dhT17/sHt2zcB6Lve2ds7QBCECtOdkIiIiMhUGJZMTHgoLZV066oTdKYrppJSJyUiedNGZBw+KG7zGTceds81krCqZ3f9+lXs27cLeXl5AABnZ1d07twd7u4eEldGREREVD4xLJmY8HA/vPtpKSWvcC6fuq6cFLQs5d2+hZj5X0ObmanfIJPBZ+y4ch2U8vPzcfDgHly+fEHcFhzcGC1atIVSyf/FiYiIiJ4W76QkJLuflg7EHBW3OVs6SlVOhZd56iQSli+D7n7Li1W1avAY/lK5H8jhzJmTYlCytbVDaGgY/PyqSFwVERERUfnHsGRiQgnPLJ25Fylu4zNLZSNl2xYk/fWnuO42aDBcunWXsCLjady4Oa5fvwJ3d0+0bx8CS0srqUsiIiIiqhDkUp48JiYGo0ePRosWLdCpUyfMmTMHOl3Jz+z89ttv6NatGxo1aoQ+ffogPDzcxNUay0PPLN0PSxYKCwCAUq7kQ/hGJggCMo4fRdKGv8Rt3m+MLddBKTk5ERkZ6eK6SqVC//5D0aVLDwYlIiIiIiOStGXp7bffRlBQEMLDw5GcnIw33ngDbm5uePnll4vst3PnTsydOxc//PADGjZsiA0bNmDChAnYvn07/P39Jar+6egeblm63w3vblasRNVUbLr8fMR9vxjZkecAADKlEr4TJsOmTl2JK3s6giDg339P4PDhg/D09EKfPoMgl+u/77CyYkgiIiIiMjbJWpYiIyNx6dIlTJkyBfb29ggMDMSoUaOwdu3aYvvm5eVh0qRJaNKkCVQqFQYNGgRbW1ucOXPG9IUbkwwo0BaIq009n5OulgpGm5ODO3O+EoOS3Noa3mPHlduglJGRjlWrVuHgwX3Q6bS4dy8eSUmJUpdFREREVKFJ1rIUFRUFX19fODoWDmgQFBSE6OhoZGVlwc7OTtzep0+fIu/NyMhAdnY2PD09DTqnXC6DXC5tNzeFovD8CrkcahSGJW87DyiVkvaMrBDy4+Nwd9FC5MfEAACsqlRBwLQPobC2lrgywwmCgEuXLmDfvt0oKNBfK+7unujWrQdcXFwlro7MmUIhL/JfoifhNUOG4jVDhiiv14tkYSktLQ0ODg5Ftj0ITqmpqUXC0sMEQcD06dMRHByM5s2bG3ROFxdbyZ8JkqsKP3JrGwvIrQqf0arpWQXOzrZSlFVhxG3fiRs//CiOpOHesT2qvfE6lDY2EldmuJycHGzdug0XLuhHupPJZGjbti06dOgAhUIhcXVUXjg4lL8vCUhavGbIULxmyBDl7XqR9JklocikQ0+mVqsxdepUXLt2DatWrTL4fCkp2ZK3LGVkF7Yk5eUW4NjNy+K6Nk+G1NRsKcqqEFL3RCBu5Qpx3WPgILj2fB6Z+QKQX74+18zMDKxd+wtycvR1Ozo6YcCA/nB0dENGRp7E1VF5oFDI4eBgjYyMXGi1nOyanozXDBmK1wwZwhyvl9I0UkgWllxcXJCWllZkW1paGmQyGVxcXIrtn5eXh7FjxyI3Nxe//PILnJ2dDT6nTidApzMsoBmbWlN4cQiCgH13DovrgfYB0GjM4+Ipb9IP7kfCqpUA9AM5eI95C3bPNYJWK+DhEQjLCysrW7i5eeD27WjUq9cA7dt3gqenM1JTs3mNkEG0Wh2vGTIIrxkyFK8ZMkR5u14kC0v169dHXFwcUlJSxHAUGRmJGjVqwNa2aMoTBAETJ06EUqnEihUrYGlpKUXJxiE8PHS4DGl5hUNAy2Xlqw+nuUj9ZycS//gNACCztILf5HdhXa26xFUZTqNRQ6lUAdBfGyEhXXHvXgICA6vxWTYiIiIiCUh2B1avXj00aNAAc+fORVZWFq5fv47ly5dj6NChAICwsDCcPHkSALB582Zcu3YN33zzTfkOSvjv0OFAgU4NAPCwcZOmoHIubd/ewqBkYQHf8RPLXVDSarU4ceIIfv99FfLzC7vY2djYIjCwmoSVEREREVVukj6ztHDhQnz00Udo06YN7OzsMGTIEAwbNgwAEB0djZycHADA+vXrERMTU2xAhz59+mDGjBkmr9tYNNCIyw3c6klYSfmUtmc37v2yGgAgt7WF/7tTYelXvubdSktLRXj4dty7Fw8AOHx4Pzp16ipxVUREREQESByWvLy88OOPP5b42uXLhQMfrFy50lQllbmHB7XI0qSJy9YKTipqiMx/T+Her2vEdZ+33ilXQUkQBERFncPhw/ug0ehDs7e3Lxo3NmyERyIiIiIqO5KGpcro4QEe7mTfEpdru9SQopxyKTUiHIlFgtLbsKlVW8KKDJOdnYU9e/7B7ds3AQByuRzNm7fGc881hVzOZ5OIiIiIzAXDkonpHmpZUqoKhzH3sfWWopxyJ+mvP5GybYu47jNuPOyeayRhRYa5fv0K9u4NF59NcnFxRefO3eHm5iFxZURERET0XwxLJvbwAA9Z2jRx2UpZvgeuMIWUbVsKg5JCAb8Jk2FTt3w963XvXrwYlIKDm6BFizZQKvm/IREREZE54l2aiQkPpSWdoJWwkvJD0OkQu2gBsiPPidv8Jk6BTZ26Elb1dJo3b42UlBQEBzeGn18VqcshIiIiosfgAxIm9nA3vNi8uxJWUj4IOh1iv/2mSFCq8vFn5SIoabUaHDlyACkpyeI2hUKJnj37MigRERERlQNsWTIx7UMtSw4qB6QUJEEl519DSXR5uYhd/C1yLkYBAJQurvCf+iFU9ycxNmdJSYkID9+OlJQk3L17C/37D4VCoZC6LCIiIiIyAO/STezhlqWb2TcAADWcOPHofwmCgJhv5iP36hUAgGWVAPiOnwilo5O0hT2BTqfD2bOncOzYYeh0+m6WVlbWUKsLoFBYS1wdERERERmCYcnEsnMLJ6J1sXBDSkESUvJSJazI/AiCgHurV4pBSeHoCP/3pkJuZd5hIyMjHREROxAbGwMAUCqVaNWqPerXD4ZMJnvCu4mIiIjI3DAsmZhK+dBjYjJ9K1MtZ86x9IA2KwvxK35C9pnTAACVhyf83jXvoCQIAi5fvogDByKgVhcAANzdPdG5c3c4O5t/l0EiIiIiKhnDkoQEQT9BrZytDgCArNOnEL/8J+hycgAAKi8v+E95H0onZ4kre7zY2LuIiNgBAJDJZGjSpAWaNGnBZ5SIiIiIyjmGJQnpHoSlSj4ooSAIiP9pKTKPHhG32dRvCO833oTC2nxblB7w8fFD9eq1kJR0D6GhYfDy8pG6JCIiIiIyAoYlU3togAcd9MuV/XmWhBU/FwlKzt26w23gC2b7uajVaqSnp8LNzQOA/u+vY8fOkMvlUKksJK6OiIiIiIyFYUlChd3wKm/LUubJE8g4dACAfiAHv8nvwdLHV+KqHi0hIQ7h4dtRUFCAwYNHwsbGBgBgaWklcWVEREREZGwMSxLK0mQBqLxhKWX7ViStXwcAkFtZwf+9abDw9JK4qpJptVqcOnUMp04dg3C/dfDy5Sg0atRM4sqIiIiIqKwwLEmmsDtetjpbwjpMT9BocG/tr0jfEyFu835znNkGpdTUFOzevR337iUAAFQqC7Rr1wm1a9eTuDIiIiIiKksMSyYmRiRF4XxLPrbektQiBZ1ajbtzZyPv2lX9BoUCPmPfhm1QfWkLK4EgCIiKOovDh/dDo9H/fXl7+yI0NAwODo4SV0dEREREZY1hSSIyq8LWJGtl5XjeRRAEJK37XQxKVtWqwfedSVDY2UlcWXGCIGD79o24efMGAEAul6NFizYIDm4CubxydpskIiIiqmwYliQie6hlycvWQ8JKTEMQBCSs+FkczMHS3x/+738ImZnORSSTyeDh4YWbN2/AxcUVnTv3gJubu9RlEREREZEJMSyZ2INueHKbTHGbncr8WlaM7d6va8SgpHL3gM9b75hdUNLpdEVajRo3bg6VygJBQQ2hVPJ/FSIiIqLKhv2JJCIIhXMIOVjaS1hJ2RJ0OiSsXon0PbsBAEpXV/hP/QAqM2uliYm5g99+W4HExHviNrlcjuDgxgxKRERERJUUw5JEZHKtuKySV8ybcV1+PmK+mYf0fXsAAHJra/i/OxVKRydpC3uIRqPBoUP7sHHjOqSnpyE8fBu0Wu2T30hEREREFV7FvEsvB+R2aQAAG6W1tIWUEUGnw73f1iAn6jwA/TNKPm9PgMrFVeLKCiUlJSI8fDtSUpIA6CeWbdasNRRm1j2QiIiIiKTBsGRq9x9aklnmAgByNLkSFlN2Elb8jIzDBwEAKg9P+E/7CHILC4mr0tPpdDh79hSOHTsMnU7fiuTvH4CQkG6wta34z48RERERUekwLElEUFsCyJK6jDKRvGlDYVDy9IT/u9PMJihlZKQjImIHYmNjAABKpRKtW7dHUFAwZDLZE95NRERERJUJw5JEHsyzVNOpmsSVGFdaRDiSN20AACjs7eH/3gdQOprPBK7JyYliUPLw8ERoaHc4O7tIXBURERERmSOGJYnILPIBAA4WFWckvJSd25G0bi0AQGZhAf/3PzSroAQAVavWQL16DWBjY4smTVrw+SQiIiIieiSGJRMT7j+0JJPp/5utzpGyHKMQBAH31qwqHPXOygq+E6fAwstL4sqAW7duQKPRonr1muK2Dh06s8sdERERET0Rw5IUZDpx0c26fHcBK0i8h9hFC1AQGwsAkNvYwP+9abD085e0LrVajcOH9yEq6hwsLCzh4eEFe3t9Kx6DEhERUcU3a9YMFBQU4KOP/id1KVSOMSxJ4aGw5G0nfevL08q/ewe3Pv1IXFc6u8D/vWlQuUs74WxCQhzCw7cjPT0NAKBQKJCVlSmGJSIiIjIvEye+hbNnTwMAtFotdDodVCqV+Pqvv66Hl5e3Qcd8//3pz1zXmDGv4OLFKPz111a4urqJ2+PiYjFoUG/88sufCAgILPKeJUsWISoqEt9+uxQA0LZtUyiVSsjlD6Y3lcHDwwNhYT0xfPgoKJWFt+MnTx7Hr7+uwoULUVCrC+Dt7YMuXcIwdOgIWPxnsKxdu3Zg/fo/cP36NcjlMlSpEoC+fQeiZ8/epf75du/+B6tW/YzY2Fj4+1fBmDHj0Lx5yxL3jY2NwTfffI3IyHOQyYD69RvinXcmw9fXD4B+7srvvvsGO3Zsg0ajQfPmLfDeex/CwcHx/utqzJo1C8uXL8ecOd+gZcvWpa5TSgxLErOUm8cocYbKvXYVd+d/La7LLCzgP+1DSedR0mq1OHXqGE6dOgZB0HdzrFq1Bjp27AJr64o5nxUREVFFMH/+YnH5p59+wLFjR7B06QrpCgIQHX0D0dHX0bx5S2zfvgXDh4966mN99dU8MRzodDpcuBCFDz98FwqFAiNGvAwA2LZtM+bPn4N33pmEGTNmwcrKGleuXMK8ebNx6tQJzJv3rRisfvxxCTZt+hvvvvsBWrVqA41Gg8OHD2DOnC8QHx+HV19944k1Xb16GTNnfoqZM2ejceNm2LcvAh98MAW//roeHh6exfb/5JNpCAioij//3AwAmDdvFj755AMsW7YKAPDDD4tx6dJFrFz5G1QqC8yfPwubNv2N4cNHITc3F+PHv4k6dWqJ92jlhfzJu5BRCQBkhRdJeewSVnDvHu7OmwMhPx+QyeA2aDBqfrdU0qCUmpqCv/76HSdPHoUgCFCpLBAS0g1hYb0YlIiIiCqAtm2bYu3aX9CnTzesXr0CAPDPP9sxfPggdOnSDoMG9cbff/8p7j9z5qf45JNpAPRB5KWXhmL79i0YOLAXunRpj08+mQaNRvPI823ZshFt2rRHly5h2LZts9F+Drlcjvr1G6Bfv4HYv1//vHd2dhYWLPgaY8e+g169+sLGxhZyuRx16tTD7NkLcOnSRWzZshEAcPv2Laxa9TOmTfsY7dt3hEqlgrW1NUJDu2L69P+JYWTixLcQEtK6xD8AsHnzBrRs2QatWrWFpaUlunbtjmrVamDnzu3FahYEAVeuXEZoaFfY2NjAxsYGnTt3w9WrlyEIAvLz8/D33+swfvwUuLt7wMnJCZ999qUYMHNzc9CrV298+eWXRvscTYUtS5IoDEtyWfnKq/l3buPO3NkQCgoAAO5DhsE5tIukNanVBfjrr9+Rn58HAPD29kVoaJjY7EtERFTZ5eRpEJeSbdRjKhVy2GfkIzMzDxqtrtjr3i62sLEy7q3mgQP7sHz5r3B2dkFsbAxmzPgEc+cuQtOmzXHq1AlMmjQODRoEo0aNmsXeGx8fi8uXL2L16j8QFxeL114bgX379iC0hPsYtVqNnTu3Yfr0zxAc3Ahz5nyJs2fPIDj4OaP9LA8HtePHj0Kn06JXr77F9nNyckLXrmHYu3c3+vYdgAMH9sLLywetW7cttm/btu3Rtm17AEVb60py+fIltGrVpsi2WrXq4NKlqGL7ymQytGrVBtu2bUZQUAPIZDLs2rUDrVu3hUwmw+XLl6DRaHDjxjVMn/4+cnNz0K5dB7zzzmRYW1vDxcUV/foNfPKHYoYYliQgU2jF5fLUFJl/5w5uzfgM0Orrd+nxvORBCQBUKgs0bdoSR47sR4sWbRAc3OShfsFERESVW06eBu8tOYyc/Ee3opQFG0slZr/Z2qiBqVOnznC535PF29sHW7aEw8HBAQDQtGlzODu74PLliyWGpZycHIwePRbW1taoVq06qlevgVu3oks8z8GD+6BQyNGsmX6akQ4dOmHr1o1GCUsajQZRUZHYuPEvsQteTMxdeHl5F3l+6WFVqgTg2LGjAPTPDvn7V3nmOtLT02Fv71Bkm4ODA27evFHi/h988AkmTBiLHj1CAADVq9fE/PnfAgASE+8BAI4dO4KfflqFlJQUTJ06CUuXfofx4yc/c61SYliSgqLwl5VKoXrMjuZDm52NmIXzxaDk2rc/XJ8v/QOExpacnFTkQcuGDRshIKAqnJycJauJiIiIytbDgzzIZDJs2PAntmzZiKSkJAACCgoKoFYXlPheR0cn2NjYiuuWllbIz88vcd/Nmzeic+du4nyM3br1wAcfTMGECe/CxsbG4LqnTp0kfpGr1Wrh5OSEESNG4YUXhor7aDTaR70dgiDgwZMbMpkMOt2j9zVM6b+0//jjaQgMrIZ58xZDJtM/N/XuuxPw448rIQgCNBoNXn/9TTg4OMLBwRFDhgzH8uU/MiyRYQQActt0cd3Z0vy7iunUBYhZOB+a1BQA+hYlqYJSfn4eDhzYgytXLqJfv8Hw9vYFoP/FwaBERERUnI2VvoWnTLrh2VuZtBvew5PJb9myAWvWrMRXX81FcHAjKBQK9O/f85HvLW2vk/j4eJw8eQxnzpzCpk1/i9tzc3MREfEPnn++rzhSX15eXrH3Z2dnwdLSqsi2hwd42LjxLyxb9j3CwgprrVIlEAkJccjPzyv2XkD/nNKDUff8/Pxx6NCB+wHq0c++PzzC4H9FRByGs7MT0tPTi2zPyEgv8X4qOvoGTp48jg0btsPJyQkAMGbM2wgL64irV6+IrX0Pjzzs7e2DtLTUJ9Zp7hiWJOZpI+0w208i6HSI/XYh8q5fAwDYPtcIrv0GSFJLTMxt7N69E1lZmQCAY8cOoW/fFySphYiIqDyxsVKiuo9xv6BVKuVwdrZFamo2NJriYamsXbgQheDg59C4cVMA+l4nSUmJz3zcbds2ISAgEF9+ObfI9t9/X4OtWzfh+ef7wtnZBfb2Drhy5RJq164j7iMIAi5cOI+mTZs/8vi9e/fDjh1bsHDhPEyf/hkAoFmzFrC0tMKGDesxePCLRfbPzMxEePhOTJjwLgCgXbuOWLJkEXbt2omuXcOK7Hv8+FEsW/Y9vvtu2ROfWapdux4uX75YZNvFixcQGtq12L46nf7vV6stbNF6uAUvMLAqZDIZrl69giZNmgHQD6/u4eFZroMSwNHwpPFQNzxLhaWEhTyeoNMhbsli5ESdBwBY164D7zfGmvyi12g0OHRoHzZu/FMMSrVq1UX37n1MWgcRERGZD29vH9y6dRMZGRmIj4/DggVfw9PTG4mJTx+YdDodtm3bjJ49e8PPz7/In/79X0Bk5DncunUTCoUCw4aNxLJlS3Dy5HFoNBqkpqZg4cJ5SE5OxrBhLz3yHDKZDO+++wHCw3fi2LEjAABra2tMmDAFP/ywGL/9tgbZ2VnQ6XS4eDEK48a9joYNn0O3bj0AAL6+fnjppVcxe/YMbN68Afn5ecjLy8OuXTvw8cfTEBbW85HPPj2sd+++OHHiOA4fPoj8/Hxs2bIRd+7cRrdu3QEA+/btwdixrwEAAgIC4edXBcuWfY+srCzk5GTjp59+gJ+fP6pVqw4XF1e0a9cR33//LZKTkxAbG4O1a39Bjx69nvrvwlywZcnEBAGQ26WJ6+Y6Gl5BQgLif1qKvBvXAQAW3j7wfWci5CrTPmOVlHQP4eHbkZKSDEDfv7hDh86oUaOWSesgIiIi89K370CcPn0K/fv3gJeXD6ZMmYpLly7gxx+XiN3CDHXy5HEkJSWiW7fi3fmqVauOunWDsHXrRowdOx4jRoyCu7s7vvtuIe7evQMbGxvUq1cfCxd+L3ZVe5Rq1Wpg8OAXMWfOF1i1ai1sbGwQFtYTrq6uWLNmJVauXAa1Wg1vb1+EhfXAkCHDi3QjfOWV0fD19cOff67FwoVzoVSqUL16DXz88ecljpL3qBo++eRzLFw4DwkJcQgMrIrZs+eLz4RnZ2fh7t07AAClUomvv/4GixbNw5AhfSEIAurWDcLs2fPFLonTpn2Mr7/+EkOHDoBcLsfzz/cRB7DYsWMrZs+eKZ77wTNc3br1MMrkwWVJJpSn4dieUWJiptQl4Nz1ZCw+9juUXrcAAItDZktcUXFZ584ifukS6O73w7UMCITfhMlQPNQP1RSios7hwIEIsenX3z8AISHdYGtrZ9I6zIHUXR2o/OE1Q4biNUOG4jVDhjDH68Xd/cn3tmxZksL9SWldLF0kLqS4vJs3EbfkWwhqNaBQwLlrGFx794FcZWHyWhwdnaDT6aBUKtG6dXsEBQWX+36vRERERFR+MCxJQh+WzK0LniY9DbGLF4pByW/CZNjUrWey8z9o5HwQiPz8qqBt207w9w+As7P5BUsiIiIiqtjM6269UhDEliVzCkvarCzcnfe1ODy4W78BJg1Kubk52LFjMy5ciCyyvWHDRgxKRERERCQJtixJQGaln+dAbiZZVRAE3Prfx9Ck6IOSY6cQuIT1MNn5b968gT17/kFubg7u3LkJPz9/ODpyziQiIiIikhbDkgRk8vtj1QvGmn352SQs/0kMSkoXV3gMHW6S86rVBTh0aD8uXDgnbqtTJwg2NpVvAAciIiIiMj8MSxIQNPohFjPV0o/Ol7ZvLzIOHwQAKOztUeXDjyEr5QzXzyI+Pha7d+9AenoaAMDGxhadOnVFQEDVMj83EREREVFpMCyZmCBAfGapil0VSWtJ278X99asFNerfPgxlI7Gnd37v7RaLU6dOopTp46LAzpUq1YTHTp0hrW1dZmem4iIiIjIEAxLEpA76CdYVcql+/jTDx3AvVUr9PXY2MD/3WlQubmX+XlzcrJx9uy/EAQBFhYWaNcuBLVq1eWQ4ERERERkdhiWJCDkW0NmlYtsdbYk58+/cxsJy3/Sr8jl8H17Iiz9/U1ybnt7B7Rt2wmXL0chNLQ77O0dTHJeIiIiIiJDMSxJQdC3olgrTd/tTJefj5hvvxHXfSdMhnXNmmV2vuzsLERHX0f9+sHitjp1glCnThBbk4iIiIjIrDEsmZgAiM8sOVk4mfTcurxc3F0wD5pkfTdA5+49YVsvqMzOd+3aFezbtwv5+flwcHBAlSr6wRsYkoiIiOhhEye+hbNnTwPQP9+s0+mgUqnE13/9dT28vLwNPq5Wq8W6db9hyJDHj/T76acfIjx8J378cSXq1i16b9S2bVN8/fVCtGzZusj2DRv+xJo1K/Hnn5sBAAMH9kJi4j0oFApxHxcXV3To0AmvvjoGNjY24vZLly5i1aqfce7caeTm5sLV1Q0dO4ZgxIhXYG9vX+Q8x48fxS+/rMKlS1HQ6QT4+Pige/fn8cILwyAv5aBcp06dwPffL8KtW7fg4eGJkSNfRteu3UvcNySkdbFtBQUFWLToB8hkMkyaNK7Ia4IgQK1W4+DBkwCAiIhwrFy5DDExd+Ho6ITQ0K4YPXoslEqLUtVqbhiWpHA/LClMPCntzY8/FIcItwmqD7d+A8rkPPn5eThwYA+uXLkobktKShTDEhEREdHD5s9fLC7/9NMPOHbsCJYuXfHMx7169TJ+/XX1Y8NSRkYGDhzYi5CQLti6dVOxsGSIiRPfRd++AwHoQ0R09A188sk05Obm4r33PgQAnDhxFNOmTcGoUa9h6tTpsLd3wK1bN7F48QK8+earWLp0OWxsbAEAmzdvwMKF8zBx4ruYNWseZDIZzpz5F7NmzcC1a1cxffpnT6wpKSkJU6dOwvjxU9ClSxjOnTuDqVMnoUqVANSpU6/Y/hERh4usnzt3Bp9//jHq1QuCpaVVsddXrfoZ165dBaAPgTNnfoIZM2ajRYtWuHnzBsaPHws3N3cMG/ai4R+oGWBYkoDcMk//XxOFJZ1ajej3J0ObkSFu8xn3TpkMER4Tcxu7d+9EVpZ+WHQ7O3uEhobB19c0z0QRERFRcbmaXMRnJxr1mEqFDEk6a2Rm5EKjFYq97mXrbrRHDuLj4zB//mycP38OWq0Obdq0w6RJ78HW1g55eXn4+usvcfToYeTn56N69RqYMOFd6HRavPnmq9BqtQgJaY2vv16Ixo2bFjv2P/9sQ61adTBw4GC8994EvP32RFhaWj1zzTKZDNWqVcfw4aOwaNF8vPfeh9DpdJgz50v07/8Chg8fJe4bGFgVX3zxNYYO7Y9Vq5ZjzJhxyMzMxMKF8zBmzDj06NFL3LdFi1aYMWMWdu7choKCAsyfPxs7d24rsYZff12PiIhw+PtXwfPP9wEANGvWAm3btsfmzRtKDEsP02q1mDdvNt58850SP5P4+HisXfsLfvrpFwCAlZUVPvlkJlq1agMAqFatBho0CEZ09HWDPjtzwrAkoVxtbpmfQ9DpELd0SZGgVHXW15CrjNsUqtFocOzYIZw9e0rcVrt2PbRt2wmWlpZGPRcRERGVXq4mFx8d/gq5mrK/73iYtdIan7ee+syBSRAETJ06GQ0aBGP9+i+Rm5uDTz/9EN9++w3ef/9D/PHHr0hJScEff2yASmWBX35ZidmzZ+Dnn3/B++9Px/fff4tNm3Y+8vhbtmxC37790aBBMOztHbF3bwS6devxTDU/TK1Wi8tXrlxCbGwMBg0aUmw/lUqFvn0HYMuWjRgzZhyOHz8KrVaD3r37Fdu3Xr36qFevPgDg/fen4/33pz/y/JcvX0StWnWKbKtVqw527971xNp37NgKlUqFkJDOJb6+bNkS9OzZG15eXgD0oS8wUN+TSKvV4vTpUzh37jSmT//fE89lrkzbD4yg1WnFZVdL1zI/3+0vPkf26X8BAEo3N1Sd9TVUrm5GP8+uXVvFoGRlZYVu3Z5HaGgYgxIRERE9k0uXLiA6+jrGjn0HVlZWcHZ2wSuvjMY//2yDIAjIysqEUqmEhYUlVCoVXnrpVfz005pSH/vWrWiEhHSBTCZDWFgPbN26ySh163Q6XL16Gb/8shJdu4YBAGJi7sLKygru7h4lvqdKlUDExcVCEATExsbAy8u7yLNbTyMjI73Y6MMODo5IT097Yv1r1qzAiBEvl/h6XFws9u3bg8GDi3ev27FjK0JCWmPatCl4/fWxxZ73Kk/YsmRi+brCb3VkZdwNL+/2LeTfjAYAWHh5w3/adChsbcvkXI0aNcPNmzfg7x+ATp26wtbWrkzOQ0RERIZ50MJTFt3w7B3KvhteTMxdaLVa9OwZWmS7VqtFWloa+vUbhEmTxqF//55o0aIV2rXriHbtOpTq2Js3b0CrVm3h4OAIAOjWrQdWrNAPTuDr62dwrfPnz8HChfPE+qytrTFw4BCMGvWauI9Op4MgCCUOeCUIhZ+jTKbf1xgePm5pHTlyCBqN5pGf5fr1f6BDh05wLeFL+LCwnujcuRuios7js88+hCAIGDhwkME1mAOGJRPTCoUtS04WjmV3ntxc3P7fJ+K6x8hRRg1KmZmZsLOzE/9H9/LyQf/+Q+Dh4cXR7oiIiMyMtdIaVR2rGPWYSqUczs62SJVnQ6Mxzk19SSwtLWFtbYNdu/Y/Yg9nrFmzDv/+exKHDh3AnDlf4J9/tmHGjNmPPW5eXh7Cw3eioKAAXbq0E7cLgoBt2zbj9dffBAAolUrk5+cVe39WVlaxHjQPD/Bw/PhRTJs2Gd269YBSqb/lrlIlAAUFBYiNjSkxjN2+fQv+/gGQyWTw96+CuLhY5Obmwtr60aFz1qwZj31mycnJGRkZ6UW2p6enwdnZ5ZHHBIA9e8LRunXbR97X7d27G+PGTXjk+5VKJYKDn0O/fgOxfv3achuW2A3PxHQo/GWilJdNVtVkZhQJSk6du8CmVm2jHFsQBFy8eB6//76iyPNJAODp6c2gREREREbl6+uH3NwcxMbGiNtycrLFbmQ5OTnQ6XRo1qwFJkyYgh9/XIm9eyOe2M1sz55wKBRKrFz5O5Yv/1X88/rrb2L79i1iq05AQCAuX75U7P1RUZGoVq3GI4/fvHlLtG3bAbNmzRBbdmrUqAV//ypYt+73YvtrNBps3vw3QkO7ANAPxGBlZYV1634rtu+NG9cwfPggZGZm4v33pyMi4nCJf7y8vFGnTl1cvnyxyPsvXbqAeo+ZPkYQBBw6dADNmrUs8fWrVy8jPj4OzZq1KLJ99erl+N//PiqyTSaTiWGxPGJYMrGHm0HLYjQ8dXIybs/4DOrEewAA2wYN4f7CUKMcOycnBzt2bMKePf9ArVbj+PEjyMsz7cOiREREVLnoR1RriG+++RppaWnIzMzE7Nlf4PPPPwYATJ/+Hr79dj5ycrKh0+kQGXkWjo6OsLd3gKWlJbKyspCUlFisdWjz5g3o2jUMVaoEwM/PX/zTr99ApKWl4vjxowCA4cNHYd2637B3727k5+cjMzMTa9aswMmTx8XWp0cZP34yrl27io0b/wKgDw5TpkzDpk1/Y8mSRUhLS4MgCLh16yYmTBgLOzs7DB06AgBgY2OLd96ZjJ9++gGrVy9HdnYWCgoKcOTIIUyZMh6tW7crNidTSbp27Y64uDhs3rwB+fn5OHLkII4cOYTevfsDAC5cOI9hwwYUGYgiLi4WmZkZ8PHxKfGYV65chqOjY7HHLp57rjEiInZhz55waDQa3LhxHRs2rEebNu2fWKe5Kr8xr5x6uGWpLJ5ZuvXpdOhy9QHGoV17eI582SitPTdv3sCePf8gNzcHAODk5IzQ0O6wsjLOkKBEREREj/LJJzMxb94sDBrUCxYWFmjSpDk+/PBTAMB7703HnDlfoG/fHpDJgGrVquOLL+ZCLpejSZPm8PHxwQsv9MGHH36K0NCuAIDbt2/i3LkzmDjx3WLncnBwRNu2HbB16ya0bNkaXbqEwc7ODqtWLcdXX82ASqVCzZq1MH/+d6hSJeCxdbu4uGLMmLewZMlCtGnTDu7uHmjSpBm+++5HLF/+I4YNG4C8vDy4u7ujU6fOGDnyZVhZFQ7R3bNnb7i6umHNmhVYvXoFZDLA3z8Ao0ePRVhYz1J9ds7OLpg9ez4WLPga8+bNgpeXNz7++HPUqFETgL474u3bt4o8H5WSkny//pIHBUtJSYaLS/GByho0CMZnn32BpUu/w+effwxnZxd07twNI0e+UqpazZFMeJonvsqpxMRMqUvA7qgL+CthBQDg5dqj0NT38ePbGyLj8CHE//wjAMDCxxcBn8145qCkVhfg0KF9uHAhUtxWv34wWrVq/8yjs1Dpif3CU8u2XzhVHLxmyFC8ZshQvGbIEOZ4vbi7P7llji1LJlZ0lBPjtSxlHj8mBiUA8Jv83jMHpaSkROzYsUl8KNDGxhYhIV1RpUrVZzouEREREVF5wLBkYnm6HHFZEIyTqgvi4xC3dIm47vfeNCgdn32kPSsrK7F/b7VqNdGxY2d2uyMiIiKiSoNhycRkD42pYaO0eebjaXNyEPPtN+K6S68+Rhv5zs7OHh07doVGo0atWnU50h0RERERVSoMSxKSGyF8JP31J9Tx8QAA57AecOvT76mOIwgCIiPPIDU1GR06dBa3V69e85lrJCIiIiIqjxiWTMyYo2nk372D9H17AAA2QfXh1n/gUx0nKysTERH/4O7dWwAAP78qqF69ltHqJCIiIiIqjxiWTO6huPQMLUu6/HzEfDMPEARAoYDHsOGQyQ0fMOLq1cvYvz8c+fn5AABXVzc4OTk/dV1ERERERBUFw5KEZHi6sKROSsSdOV9Bk5oKAHAJ6wELTy+DjpGXl4cDByJw9WrhjNSNGjVF8+atoVDwsiAiIiIi4l2xiQnP2BFPnZqKW59+BF2efpQ62+cawdXA55Tu3r2N3bt3IDs7CwBgb++A0NAw+Pj4PVNtREREREQVCcOSiT08BbChLUva3FzELJgrBiX75i3gOfJlg7rf6XQ6HDq0VwxKdeoEoW3bjrCwsDSoFiIiIiKiis54s6JSmRJ0OsT9sAQFMXcBAE6du8B79JuQW1kZdBy5XI7Q0O6wtbVFt269EBLSjUGJiIiIyoWBA3thw4Y/AQDjxo3GkiWLJK6IKjq2LJlcYdOSIeM7JKz4GTnnzwEALAOrwn3QkFK9T6fT4dKlKNSuXQ8KhQIA4ObmjuHDX+WzSURERGR2jh8/ikmTxqFfv0GYPPn9pzrGTz/9gBUrlkGlUkEQBFhaWqFWrdp4/vk+6Nq1e6mPc+rUCdja2qJOnXpPVQcAbNz4F+bM+QJjx76DYcNGFnlt3LjRCApqgDfffLvI9lu3buLFFwdi3bpN8Pb2wcyZn2Lnzm1QKgvv3Wxt7RAc/BzefPMd+PoWPkqRlJSElSt/wuHDB5CamgpbW1s0bdocL7/8OqpUCShyntu3b2L58mU4efI4cnKy4eLiijZt2uOVV16Hg4NjqX6++Pg4zJ37FaKizsPa2hqhoV0xZsw4yJ/Q8ykx8R6GDRuIIUNexKuvvgEASEtLw6JFc3H8+DGo1WrUqlUbb701AbVr1xHP9c03c3H27GkoFAq0bNka77wzGfb29qWq9WmwZcnEij6xVLq0lLThL2QcPggAsPD1g9+kdyG7H3weJz09DRs2/IG9e3fh1KljRV5jUCIiIiJztHnzBoSGdkV4+E5xtN6nUbduECIiDmPPniNYs+YP9OrVF4sXL8CsWTNLfYy1a3/BpUsXnroGANiyRf/zbNu2+ZmO06lTKCIiDot/Vq78DXK5Au+/PxFarRYAkJSUiNdfH4mcnCwsXPg9IiIOYcWKX+Hh4YnXXx+JK1cKB/a6evUyXnvtJbi7e2Dlyt/wzz/78cUXc3Dt2hW8+earyM/PK1VdH3zwLtzcPPDHHxuxYMF32L9/L/7449cnvm/BgjlQKIpGkblzv0JKSgrWrPkDmzbtRFBQA7z77njx53v//Ymwt7fHn39uxk8/rUZ09A0sXryglJ/g0+Eds8k91LJUir0zT51EypZNAACFoyP8Jr8HhY3N488gCLh48TwOHdoLtVoNQD+oQ9OmLZ+Y8omIiKji0ebkoCA+zqjHVCjkUNpbISczD1qtrtjrFl7eT7xn+a/09DQcOrQfK1f+jsuXL2H//j3o0iXsmWt1c3NHly5hqF69Bl5++UV06dINjRs3RX5+PhYsmIPDhw8gNzcPNWrUxJQpU1GtWg28//5EHD58EMeOHcGePbvxzTdLcOnSBSxcOA83blyDSmWBDh06YcKEd4u0+Dzs+vVruHHjOmbPXoDBg/vh/PlzqF+/4TP/PADg4uKKceMmoH//nrh9+xaqVq2GH35YDDc3d3z00efifq6ubnjzzbeRkBCPuXNn4YcflgMA5s2bjebNW2Ls2HfEfWvWrI3Zs+djwYKvkZSUhMjIs5g9u+Rw+d57HyIwsCquX7+KBQu+g52dHezs7DBkyDD88cdvGDJk+CNrP3LkIG7ejEbr1u2KbL98+SKGDh0BR0cnAEBYWE+sXr0cyclJsLa2Qe3adTFmzDjY2NjAxsYG3bv3xLp1a5/2IywVhiUTE4p0w3t8XNJmZSH+52Xiut+kd6F0cHjse3JycrBv3y5ER18Xz9GsWSs0btycQYmIiKgS0ubkIHrqFOhyckx6XrmNDap+9bVBgWnHjq2oUaMW/P2roGvXMGzZsskoYemBatVqoFmzFoiICEfjxk3x66+rcOHCeaxe/QesrW0wb94szJjxKX7+eQ1mzZqPgQN7Yfjwl9C370AAwMcfT0O3bj2waNEPSExMxJtvvoLAwKoYOLDkxyM2b96Atm3bw9nZBR07hmDLlo1GC0sAxC/FAf2jF/v378HkyVNL3HfQoKF4441RSEy8B6VSicjIs1i06Idi+9nY2OKDDz4BAPj6+iEsrOcjz79hw3p4eXnD4aH701q16uD27VvIycmGjY1tsffk5eVh3rw5mDbtI2zfvqXIa61bt0N4+E60a9cRtra22L59C2rWrAV3dw/IZDKxrgcSEhLg7u7+yPqMgXfPEnpcVBJ0OtyZ8xWE+02gHiNGwdL38UN737x5HWvXrhKDkpOTM/r3H8oWJSIiIioXtmzZiG7degAAunXrgdOnTyIuLtao56hSJUA85ogRL2PJkp/g4OAIlUqFTp064/r1q9BoNCW+d8WKXzFy5CtQKBTw8vJCcHAjXLp0scR9CwoK8M8/29Gtmz5shIX1REREOHJzc43ycyQm3sO3385HrVq1ERhYFWlpqcjOzoafn3+J+z94Xikm5i5iYmKKbHtaGRnpsLcv+kX+g2ed0tLSSnzPzz//iPr1G6Bx46bFXnvrrfFQqVTo2zcMXbrog9Onn35RYgPDpUsXsH79Wowc+coz/QxPwpYlM3Xnq5niyHfWderCqUPHx+4fFXUO+/aFi+v16wejVav2UKlUZVkmERERmTnF/RaesuiG52BvhQwjdcM7fz4Sd+7cRmhoVwD6Vo2goAbYtm2zOACAMWi1WvFZmdTUFCxYMAdnzvyLnJwcCIIArVYLrVZbYte6kydPYMWKH3Hnzm1otVpoNBp06hRa4nn2798DuVyO5s1bAgAaNWoCOzs7RETsQs+evQ2ue8+e3ThwoDUA/SMXarUa3bs/jylTphUJEw+e7/kv4f78NTKZTNxfpyv+92YoQSj9HKLXrl3Dpk1/Y+XK30t8fe7crwAA69dvgZ2dHdat+x0TJ76FNWvWweaha+ncuTN4//1JGDPmbTRr1uLZfoAnYFgysSIX1CO64WVHnkPejeviuvfrY5543GrVauLEiSMAgJCQbqhSJfCZ6iQiIqKKQ2FjA+tq1Y16TKVSDntnW2hSs6HRPPtN95YtG6DVajFoUC9xm1qtRmLiPbz88utG6yVz5cplBAU1AAB88skHsLCwwPLl+kEQTp48jgkTxpb4vlu3buKjj97HuHET0bt3X1haWuHzzz96ZCvU5s0bkZmZge7dO4nbCgoKsHXrJjEsqVSqEgdSyMrSz4dpaVk4vUunTqH47LMvAQApKckYNmwgmjZtDldXNwCAs7ML7O0dcOtWNBo0CC52zNu3bwHQtyY9+Cyjo2/A3d3jkZ/Vjh1bH/vMkpOTMzIy0otsT09Ph0wmg5OTc5HtgiDg008/xWuvvSHW/LDc3Fxs3boJ3323DJ6eXgCAl156FWvX/ooTJ46iQ4cQAMDBg/vx+ecfYcKEd9G9+/OPrN1YGJYkVFJUErRaxC5eKK77f/AxlI7Fh2588G3Gg/+JrK2t0aNHHzg4OMLKyrqsSiYiIiIyupycHOzevQtTpkxDkybNxO25ubkYPfolnDp1HM2atXzm85w4cQyRkWcxfvwUAMDFi1H46KPP4eHhCQBFRov7rytXLsHCwgKD7k/fIggCrly5jGolhNDY2Bj8++8JzJo1v0hXt7i4WEyc+BZu376FKlUCEBAQiMuXi58zKuocXFxc4ezsUmItLi6uGDNmHBYunIvmzVvB2dkZMpkMHTuGYP36P9CzZ59iXdf+/nsdGjduKh6zUaMm+P33NWLL1wN5eXkYO/Y1TJgwBWFhPR/7zNKVK5eQkBCPtLQ0ODk5AQAuXYpCYGDVIi1BgH7Y7xMnTuDKlav48cfvAej/fmUyGQ4d2o9Fi36AIAhFWrsEQSgSRiMjz2LGjE/w+eezitVdVvggi4k93FApKyEupe0Oh3D/onAfPBTW1aoV2yclJRl//fUbIiJ2Fmmp8vDwYlAiIiKicici4h9YWlqgR49e8PPzF//UrFkLbdq0x5YtG5/p+Pn5+YiICMdnn03H0KEjUKdOXQCAl5c3Llw4D41Gg6NHD+P48aMA9ENwA/qWnZiYGGRlZcHb2wf5+fm4evUyMjIysGTJQqhUFkhKSirWFW3r1k2oVq0GWrduW+TnadasBerUqYetW/UjHQ8YMBjXrl3FTz/9gMzMTOTn5+PAgb1YvnwZxo2b+NjBwPr06Q9//wAsWDBH3Pbaa2OQnp6Od98dj7t37wDQ3zfqR/w7iEmTCuetGj9+CqKiIvHJJx/g3r0E6HQ6XL16GZMnvw0rKyvUq1f/iZ9rrVp1UKdOPXz//SJkZ2fh1q2b+P33X8UBMQBg2LABOHv2DDw8PLFv3z6sXv0bli//FcuX/4o2bdqjb98BmDPnG9ja2qFRoyZYufInpKQkIz8/D6tXL4dSqcRzzzWGRqPBrFkz8Oabb5ssKAEMS2Yl/84dJP7xGwBAZmkJp//0gRUEAefO/Yt169YgMfEeoqOv4dataClKJSIiIjIa/ah33Ut81rpnz944cGBfse5eT3LxYhRCQlojJKQ1evYMxW+/rca4cRPw1lvjxX0mTXoP+/btQffuIdiyZQM+++wL1KtXH6++OhwpKcno1asv/vrrD4wbNxr16zdE//4vYNy40Rgx4gV4eflg/PgpuH79Gj755APxmDqdDtu3b0HPnr1KKgs9e/bGjh1bodVq4e9fBUuXrsD169fw4osD0adPN6xa9TPGj5+Mrl0fPwqgTCbDu+9+gH37InDo0AEA+mHCly5dCQ8PT7zzzhiEhLTBqFHDkJmZiWXLViEwsKr4/ho1amLp0pUQBAGvvDIcXbq0w8cfT0Pjxk0xf/63jxwO/b9mzJiFpKRE9O7dDW+//QbCwnqgf/9B4uu3b99Cbm6OOCiGh4en+MfKygo2NrZit7zPPvsCTk7OePnlYejTpztOnDiGr79eCEdHJ0RFReLmzWh8883X4t/rgz/xRn4e72EywZCnssq5xMRMqUvAX2cOY3fKBgDAe40mI8BZ3+yrUxfg1icfQX0vAQDg8/YE2AU/J74vKysTERE7cffubQCAXK5Ay5ZtERzc+IlDkFP5p1TK4exsi1Qj9Qunio/XDBmK1wwZitcMGcIcrxd3d/sn7sNnlkysaDQtDDkJy38Sg5JD67ZFgtLVq5exf3+4OIu1q6s7OnfuXuLDcUREREREZBwMSyb38KS0+v/GfrcIWf+eAgBY164Dz5dfBQCo1QXYuzccV68WPvjXqFEzNG/eCgoF/+qIiIiIiMoS77hN7L8DPBQkxItBSWFnD+/Rb4rd6uRyBdLTUwEA9vYOCA0Ng4/P4yemJSIiIiIi42BYkpJGg9jFi8RV1379iwwTrlAo0LlzD5w5cxKtW7eHhYVlSUchIiIiIqIywNHwTK6wbUlz5AgKYmMBAM7dwqCuF4SdOzcXGU/eyckZHTt2YVAiIiIiIjIxtiyZ2IOopFLroN62HQCg9PTCrapVcXL9r9DpdLCzc0CbNh2kK5KIiIiIiKRtWYqJicHo0aPRokULdOrUCXPmzCkya+/DVq1ahW7duqFx48YYOnQozp8/b+JqjePBSO2hxzMBtRo5FhY4VbsGjh8/BJ1OB6VSCScnZ4mrJCIiIiIiSVuW3n77bQQFBSE8PBzJycl444034ObmhpdffrnIfhEREVi0aBGWLVuG2rVrY9WqVRgzZgz++ecf2NjYSFT90xJgn61F1bv5iHF1xhV/X2izMgAAnp5eCA3tzrBERERERGQGJGtZioyMxKVLlzBlyhTY29sjMDAQo0aNwtq1a4vtu3btWvTv3x/BwcGwsrLCa6+9BgDYs2ePqcs2iuf3ZeJCQCAuBvhDK5dDLpejefPW6NdvCIMSEREREZGZkKxlKSoqCr6+vnB8aPS3oKAgREdHIysrC3Z2dkX27dGjh7gul8tRt25dREZGomfPnqU+p1wug1wue/KOZcgqLQ2pdm5IcnIAoB/AoVu3nvD09JK0LjJvCoW8yH+JnoTXDBmK1wwZqiJcMz16dMHYsW/j+ed7S11KhVderxfJwlJaWhocHByKbHsQnFJTU4uEpbS0tCKh6sG+qampBp3TxcVWnMNIKlWdXaCIjUeSoz1qNmqE5/v2hUqlkrQmKj8cHKylLoHKGV4zZCheM2QoY10zISEhSEhIgFxeeDPt7u6OLl264J133oGtra1RzvMwuVwGW1tLODsb/9hUsvL2O0bSZ5YeDHZg7H0fJSUlW/KWpeAmbbF3QDJaurviuVYdkZVVAKBA0prI/CkUcjg4WCMjIxdabcmDoBA9jNcMGYrXDBnK2NeMVqvDpEnvoX//gQD0937R0TcwffpUpKVlYOrU6c98jv/S6QRkZ+cjNTXb6Memoszxd0xpQrJkYcnFxQVpaWlFtqWlpUEmk8HFxaXIdmdn5xL3rVmzpkHn1OkE6HTPHrqeVUjPfnB2tkVqajY0GvO4WKh80Gp1vGbIILxmyFC8ZiquhIS4J+7j6ektLmu1GiQlJT5yX4VCjuxsW1hZOYjXTEFBAVJTk4sdq7R0uqLXX5UqVfHiiy9h0aL5mDLlA1y6dAELF87DjRvXoFJZoEOHTpgw4V0olUr8++9JTJs2GZ999iUWLpyLe/cS0LBhI3z66Uw4ODhAo9Fg0aJ52LVrJywsLPDqq2/cP6cAjUaH/Px8LFmyCPv370F6ehrq1KmHCROmoGbN2gCAtm2b4tNPZ+K339YgOvoGmjRpinff/QBffTUDkZFn4O8fgJkzZ8Pb28fgn7syKW+/YyQLS/Xr10dcXBxSUlLEcBQZGYkaNWoUa2atX78+oqKi0K9fPwCAVqvFhQsXMHDgQJPXTURERFQerV//2xP3GTt2krick5PzxPc4OjripZdeF9eTkxPx999rix3rWajVanH544+noVu3Hli06AckJibizTdfQWBgVQwcOAQAkJeXh/Dwnfj+++XIy8vF66+/hM2b/8aLL76ErVs3Yc+e3fjuu2Xw9PTC4sULkJmZIR576dLvcObMv1i8+Ee4uLjg++8X4733JuKPPzaKj0xs2LAes2bNR25uDl56aQimTHkH06d/Bl9fP7z11mj8/vsaTJz4nlF+bjIPkj1hVa9ePTRo0ABz585FVlYWrl+/juXLl2Po0KEAgLCwMJw8eRIAMHToUGzYsAFnzpxBbm4ulixZAgsLC3Ts2FGq8omIiIioDOl0Oly9ehm//LISXbuGAQBWrPgVI0e+AoVCAS8vLwQHN8KlSxfF92i1WgwbNhIODg7w8PBEw4bP4datmwCA/fv3okuXMAQGVoW1tTVee+1NaDQa8b1bt27EiBGj4O3tA0tLK7z++ptITk5CZORZcZ/OnbvBzc0N/v5VUKVKIOrWDUKtWnVga2uHRo2a4M6dO6b5cMhkJH1maeHChfjoo4/Qpk0b2NnZYciQIRg2bBgAIDo6Gjk5OQCA9u3bY9KkSZgwYQKSk5PRoEEDLF26FFZWVlKWT0RERFRuDBgw1KD9bWxsHvsehUIOJ6eivYFcXd0NPs/D5s+fg4UL5wHQBx9ra2sMHDgEo0bpp405efIEVqz4EXfu3IZWq4VGo0GnTqFFjuHj4ysuW1lZIT8/DwCQmJiA1q3biK85OTnB3l4/2FhGRgaysrIQEFBVfN3GxgbOzi6Ii4sVt3l6eorLFhYWcHf3KLJeUJD/1D87mSdJw5KXlxd+/PHHEl+7fPlykfVhw4aJQYqIiIiIDGPoM0QKhfKx71Eq5eIz2A9YWFg81bNKD0yc+C769tU/ZnH8+FFMmzYZ3br1gFKpxK1bN/HRR+9j3LiJ6N27LywtrfD55x8VaR0C8MiRjwsK1NBqtUW26XT6Z2fU6kcPtvXw8WQy+SNfo4qpfA10TkRERESVQvPmLdG2bQfMmjUDgiDgypVLsLCwwKBBQ2BpaXV/2+UnH+g+Nzc3JCQkiOtJSUnIysoEADg7u8DGxlbssgfoW5tSU1PgR9nElQAAE6FJREFU6+tntJ+Jyh+GJSIiIiIyS+PHT8a1a1exceNf8Pb2QX5+Pq5evYyMjAwsWbIQKpUFkpKSSjXFTMuWrREevhN37txGTk42li5dDAsLSwCAXC5Hly7dsGbNcty7l3D/GflF8PHxRf36Dcv6xyQzxrBERERERGbJxcUVY8a8hSVLFsLT0wv9+7+AceNGY8SIF+Dl5YPx46fg+vVr+OSTD554rMGDX0Tr1u0wevQoDB06APXrN4SHR+EzR+PGTUTNmrUxevQoDBjwPJKTk7BgwXdQKBRl+SOSmZMJxpjttZxITMyUugQARfv4lqdx5kk6vGbIULxmyFC8ZshQvGbIEOZ4vbi72z9xH7YsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiIiIiIhKwLBERERERERUAoYlIiIiIiKiEjAsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiIiIiIhKIBMEQZC6CCIiIiIiInPDliUiIiIiIqISMCwRERERERGVgGGJiIiIiIioBAxLREREREREJWBYIiIiIiIiKgHDEhERERERUQkYloiIiIiIiErAsERERERERFQChiUiIiIiIqISMCwRERERERGVgGGpjMTExGD06NFo0aIFOnXqhDlz5kCn05W476pVq9CtWzc0btwYQ4cOxfnz501cLZkDQ66Z3377Dd26dUOjRo3Qp08fhIeHm7haMgeGXDMPJCQkoFGjRli0aJGJqiRzYcj1cv36dYwYMQLBwcHo0KEDVqxYYdpiySyU9prR6XRYuHAhQkJC0KhRI/Tq1Qvbtm2ToGIyBwcOHEDr1q0xceLEx+6n0+kwf/58hIaGolmzZnj11Vdx584dE1VZegxLZeTtt9+Gp6cnwsPDsXz5coSHh2PlypXF9ouIiMCiRYswe/ZsHD58GJ06dcKYMWOQk5MjQdUkpdJeMzt37sTcuXPxxRdf4Pjx4xg+fDgmTJhglr9gqGyV9pp52IwZM6BQKExUIZmT0l4veXl5eO2119ChQwccPXoUixYtwp9//onr169LUDVJqbTXzG+//YZ169Zh2bJlOHnyJCZNmoR3330Xly5dkqBqktKPP/6IGTNmICAg4In7/vLLL9i8eTOWLl2KPXv2IDAwEG+99RYEQTBBpaXHsFQGIiMjcenSJUyZMgX29vYIDAzEqFGjsHbt2mL7rl27Fv3790dwcDCsrKzw2muvAQD27Nlj6rJJQoZcM3l5eZg0aRKaNGkClUqFQYMGwdbWFmfOnDF94SQZQ66ZB/bt24dr166hY8eOpiuUzIIh18v27dthZ2eH1157DdbW1mjYsCG2bNmC6tWrS1A5ScWQayYqKgpNmjRBtWrVoFAo0KlTJzg5OeHy5csSVE5SsrS0xJ9//lmqsLR27VqMGjUK1atXh52dHSZOnIjr16/j7NmzJqi09BiWykBUVBR8fX3h6OgobgsKCkJ0dDSysrKK7VuvXj1xXS6Xo27duoiMjDRZvSQ9Q66ZPn36YNiwYeJ6RkYGsrOz4enpabJ6SXqGXDOAPmT/73//wyeffAKlUmnKUskMGHK9nDp1CrVq1cK0adPQtGlThIWFYdOmTaYumSRmyDXTsWNHHD9+HBcvXkRBQQF2796N3NxcNG/e3NRlk8RGjhwJe3v7J+6Xl5eHa9euFbkHtrOzQ0BAgNndAzMslYG0tDQ4ODgU2fbgl01qamqxfR/+RfRg3//uRxWbIdfMwwRBwPTp0xEcHMx/lCoZQ6+ZxYsX47nnnkPLli1NUh+ZF0Oul/j4eOzevRutW7fGgQMH8MYbb+D999/HhQsXTFYvSc+Qa6Zr164YPHgw+vbtiwYNGmDy5Mn48ssv4e3tbbJ6qXxJT0+HIAjl4h6YXy+WEUP6W5pb30yShqHXgVqtxtSpU3Ht2jWsWrWqjKoic1baa+batWtYt24dNm/eXMYVkTkr7fUiCAKCgoLQq1cvAEC/fv3w+++/Y8eOHUW+BaaKr7TXzIYNG7BhwwasW7cOtWvXxpEjRzB58mR4e3ujYcOGZVwllWfl4R6YLUtlwMXFBWlpaUW2paWlQSaTwcXFpch2Z2fnEvf9735UsRlyzQD65us33ngDsbGx+OWXX+Dm5maiSslclPaaEQQBn376Kd5++224u7ubuEoyF4b8jnF3dy/WjcbX1xeJiYllXSaZEUOumTVr1mDw4MFo2LAhLC0t0bFjR7Rs2ZLdN+mRnJycIJfLS7zGXF1dpSnqERiWykD9+vURFxeHlJQUcVtkZCRq1KgBW1vbYvtGRUWJ61qtFhcuXEBwcLDJ6iXpGXLNCIKAiRMnQqlUYsWKFXB2djZ1uWQGSnvNxMbG4sSJE1i4cCFatGiBFi1aYOvWrVi2bBn69esnRekkAUN+x1SvXh1Xrlwp8o1vTEwMfH19TVYvSc+Qa0an00Gr1RbZVlBQYJI6qXyytLREzZo1i9wDZ2Rk4Pbt22bXGsmwVAbq1auHBg0aYO7cucjKysL169exfPlyDB06FAAQFhaGkydPAgCGDh2KDRs24MyZM8jNzcWSJUtgYWHB0aoqGUOumc2bN+PatWv45ptvYGlpKWXZJKHSXjNeXl7Yt28fNm7cKP4JCQnBkCFDsHTpUol/CjIVQ37H9O7dG6mpqfj++++Rl5eHLVu2ICoqCr1795byRyATM+SaCQkJwZ9//olLly5Bo9Hg4MGDOHLkCEJDQ6X8EcjMJCQkICwsTJzqZOjQoVi1ahWuX7+OrKwsfP3116hbty4aNGggcaVF8ZmlMrJw4UJ89NFHaNOmDezs7DBkyBBxBLPo6GhxHqX27dtj0qRJmDBhApKTk9GgQQMsXboUVlZWUpZPEijtNbN+/XrExMQUG9ChT58+mDFjhsnrJumU5ppRKBTw8vIq8j5ra2vY2dmxW14lU9rfMZ6envjhhx8wc+ZMfPfdd/Dx8cHixYtRpUoVKcsnCZT2mnnjjTeg0Wjw1ltvISUlBb6+vpgxYwZatWolZfkkgQdBR6PRAADCw8MB6Fsl1Wo1oqOjxVbHIUOGIDExESNGjEB2djZatGiBb7/9VprCH0MmlIcnq4iIiIiIiEyM3fCIiIiIiIhKwLBERERERERUAoYlIiIiIiKiEjAsERERERERlYBhiYiIiIiIqAQMS0RERERERCVgWCIiIiIiIioBwxIREREREVEJGJaIiMhgx44dQ+3atXH9+nWpS3km06dPx/Dhwx/5eoMGDbBu3ToTVkREROZEKXUBRERkWiNGjMDJkyehVBb/JyAkJATffPONBFUV9ddff2HatGmwsLAosr1mzZp49dVX0bNnT6OcZ8aMGUXWN27ciOeeew4BAQEAgMjISKOc51Fq164NpVIJubzwu0sbGxvUrFkTb7zxBtq1a1fqY8XHx+PAgQMYNGhQWZRKRFQpMSwREVVCYWFhmD9/vtRlPNHJkydhaWkJAMjNzcXmzZsxZcoU6HQ69OrVy6jnEgQBX375JWbPni2GJVOYPn06hg4dKq5nZmZi1apVGDNmDH799VcEBweX6ji7du3C5s2bGZaIiIyI3fCIiKiYnJwcfPrpp2jVqhUaNmyIzp07Y8WKFY/c/8iRI3jhhRfQpEkTNG3aFC+//DKuXbsmvn7ixAmMGDECzZs3R5MmTfDmm2/izp07BtVkbW2NF154Ac2bN8fu3bvF7bt27UL//v3RuHFjtGjRAlOmTEFKSor4+i+//IJu3brhueeeQ6tWrTBt2jRkZWUBAKZOnYoXXngBmZmZaNCgAVJTUzF27Fixa17t2rXx22+/4Y8//kCDBg3E9z0wefJkDBs2DACQnp6ODz/8EB07dkRwcDB69eqFrVu3GvQzAoC9vT3eeusteHl5YfPmzeL2/fv3Y9CgQQgODkbz5s3x+uuv49atWwCAWbNm4YsvvsC5c+fQoEEDHDp0SPxsBg0aJH427777bpHPhoiIHo9hiYiIipk7dy4OHjyIv//+G2fPnsX06dPx5Zdf4sCBA8X2VavVeOutt9C/f38cP34ce/fuRdWqVTF9+nQAwPXr1/Hqq68iNDQUBw/+v737jan57+M4/jwrLakdGmomCuuGEcMUSoSMmo60UW12GmPIZg3ZiLGQZNwRy/zbsjJjNUxTjKUQh/ybMBERDSXNnzqnc90w53KuDj9n143r2rwet9r3+/l8vu/v996rz59zhYqKCnx8fEhLS6Ojo8Pt2mw2m2MJYU1NDStXrmThwoVcu3aNkydPUl9fz6pVqwCora0lJyeH3bt3U1tbS0lJCc+ePaOgoMBpTD8/P8rKygDIz8+nsLDQ6f6sWbMwGAxUVFQ4rn3+/JmLFy8yd+5cAFasWEFTUxPFxcXcvHmTZcuWsXbtWqqrq91+R7vdjs1mc8yqNTc3s3z5cmJjY7l16xbl5eVYrVbWrFkDQGZmJgkJCYSFhXHv3j0mTZrE1atXycjIwGw2U1NTQ2lpKc3NzaSnp7tdj4jI30phSUREusnMzOTUqVMEBgZiMBiYMmUK/fr1o7a2tlvbjo4Ovn79ire3Nx4eHvj6+pKVlUVxcTEAx48fZ9iwYZjNZry8vOjTpw/r16/n5cuXWCyWP66pvb2doqIiLBYLiYmJABQWFjJhwgRMJhNeXl4MHDiQ5cuXc/36dV6/fk1bWxvwfVYKICAggKKiIjIyMtz6Hn5+fsTExHDu3DnHtYqKCux2O7NmzaKuro4bN26QmZlJYGAgPXr0YPbs2URGRlJSUuLWsz58+EBubi7v3r0jISEBgP79+3PlyhXS0tLw8PDAaDQyc+ZM7t+/j9VqdTlOYWEhU6ZMIS4uDk9PTwIDA1m9ejUWi8XtWT0Rkb+V9iyJiPyFysrKnGZJfli6dCnp6em8ffuWnTt3cvPmTT59+gR8D0Xfvn3r1qdXr15kZGSQlZXF/v37mTBhAjNmzGDixIkA1NfX8/DhQ0aOHOnUz9PTk8bGxt/WOW7cOMffPj4+DB06lPz8fMfYDQ0NREREOPUZNmwYAC9evCAiIoLp06cTFxdHWFgYERERxMfHO9q4w2QykZ6ezsePHzEajZw+fZrY2Fh8fX2pr68HICkpyamP3W5n9OjRvx03Ozubbdu2Odp3dnYSGRnJkSNHCA0NdbQ7e/YsxcXFNDY2YrVa6erqwmazOc20/ay+vp6GhoZu393Dw4PGxkaCgoLc/gYiIn8bhSURkb/Q7w546OrqYvHixfTt25eioiIGDRqEwWAgOjr6l+MtXryYpKQkqqqqqKysZMWKFcTExLBr1y68vb2JiorqtvTtT/x8wIMrrsJbV1cXAAaDAS8vL3bv3s3Lly+prKzk0qVLFBQUsHHjRhYsWOBWLZGRkRiNRsrLy4mJiaG6upqDBw8COGq8fPky/v7+bo378wEPHz9+JD4+ntDQUKegWFpaSnZ2NtnZ2cyePZuePXty4sQJx1JHV7y9vZk/fz6bNm1yqx4REfk3LcMTEREn79+/5/nz56SmpjJ48GAMBgNNTU28ffv2l30+fPhA7969iYuLIycnh/z8fM6cOUNrayshISHU1dVhs9kc7W022z/OKv2J4OBgHj165HTtyZMnjntWq5W2tjaCgoJISUmhoKCAJUuWcOzYMbef5enpSVxcHGVlZZw/f56AgADCw8MBCAkJAeD+/ftOfV69euX03v/EaDSydetWjhw54rTXyWKxEBISwrx58xxLCu/cufPbsUJCQnjw4IHTtS9fvtDc3PzH9YiI/O0UlkRExIm/vz9+fn7cunULq9XKo0eP2Lx5M0FBQTQ1NXVrb7FYHIc32Gw2Ojo6qK2tpW/fvhiNRpKTk2ltbSU3N5e2tjba29vJy8sjKSmp2+ly7kpOTubatWuUlJTQ2dlJQ0MDe/fuZerUqQQEBHDgwAFSUlIcP57b2trK48ePGTJkSLexfHx8gO/L137sdfpPJpOJ69evU1payty5czEYDAAMGTKE6OhocnNzefr0KTabjaqqKubMmeO0z+lPTJ48maSkJDIzM2lpaQFg8ODBvHnzhoaGBtrb2yksLHS80+vXr4Hv+7Kam5tpaWnhy5cvmM1m7t69y6FDh/j8+TMtLS1s2LABs9nsmH0TEZHfU1gSEREnHh4e5OTkcOnSJcaNG0dWVhbp6emYzWYuXLjgOIHth7Fjx7Ju3Tq2bt3KmDFjiIqKoqamhv3792MwGBgwYAAFBQXcuXOHqKgooqKiePz4MUePHsXX1/e/qjU6Oprt27dz+PBhxo8fT1paGuHh4ezatQuARYsWMXHiRMxmM2FhYcTHx+Pr6+tyaZq/vz9z5swhLy+P1NRUl88bPnw4wcHB3L59G5PJ5HRvx44djBgxgpSUFEaNGsWWLVtYu3Yt8fHxbr/XunXr8PLyciyzS05OJjIyEpPJRGxsLG/evGHfvn2EhoaSmJhIXV0dCQkJWK1WoqOjqaioICwsjD179lBaWkp4eDjTpk2js7OTAwcOOP0IroiI/JrBbrfb/9dFiIiIiIiI/L/Rv5ZERERERERcUFgSERERERFxQWFJRERERETEBYUlERERERERFxSWREREREREXFBYEhERERERcUFhSURERERExAWFJRERERERERcUlkRERERERFxQWBIREREREXFBYUlERERERMSFfwE8d43vndg8GwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "### ASSESS MODEL PERFORMANCE ###\n", "\n", @@ -1828,13 +739,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-18T02:43:37.750244Z", - "start_time": "2024-03-18T02:43:37.449960Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "xgb.plot_importance(xgb_model, max_num_features=20)\n", @@ -1844,13 +749,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-13T15:15:59.835420100Z", - "start_time": "2024-03-13T15:15:59.795421400Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# CAREFUL, one needs to match feature ids to actual FHIR features before interpretation\n", @@ -1875,9 +774,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Save current predictions, labels, and probabilities to disk\n", @@ -1892,9 +789,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "### SCRIPT FOR K-FOLD VALIDATION ###\n", @@ -1967,9 +862,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# # Generate labels for the dataset if necessary\n", diff --git a/odyssey/models/cehr_big_bird/playground.ipynb b/odyssey/models/cehr_big_bird/playground.ipynb index 164ab0f..2be5e3f 100644 --- a/odyssey/models/cehr_big_bird/playground.ipynb +++ b/odyssey/models/cehr_big_bird/playground.ipynb @@ -2,15 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:33:04.432760500Z", - "start_time": "2024-03-05T16:32:47.338223500Z" - }, - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", @@ -38,28 +32,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "32d7b245", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:34:33.782059400Z", - "start_time": "2024-03-05T16:33:59.212297300Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_18823/310718688.py:7: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " fine_tune.rename(columns={'label_mortality_1month': 'label'}, inplace=True)\n" - ] - } - ], + "metadata": {}, + "outputs": [], "source": [ "data = pd.read_parquet(\n", " \"/h/afallah/odyssey/odyssey/data/bigbird_data/patient_sequences_2048_labeled.parquet\",\n", @@ -81,15 +57,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "c9a3eda1", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:34:37.477515300Z", - "start_time": "2024-03-05T16:34:37.378999900Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Train Tokenizer\n", @@ -101,15 +71,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "a0c5bba8", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:34:48.924095900Z", - "start_time": "2024-03-05T16:34:48.869090300Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "test_dataset = FinetuneDataset(\n", @@ -129,15 +93,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "1189c902", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:35:22.299436900Z", - "start_time": "2024-03-05T16:35:22.287440500Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "args = pd.DataFrame({\"batch_size\": [3], \"gpus\": [1], \"max_epochs\": [5]})" @@ -145,37 +103,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "bb326b6c", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:38:03.434399200Z", - "start_time": "2024-03-05T16:38:01.811400300Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:283: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.warmup = int(0.1 * grad_steps)\n", - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:284: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.decay = int(0.9 * grad_steps)\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "finetuned_model = BigBirdFinetune(\n", " args=args,\n", @@ -193,36 +124,16 @@ "cell_type": "code", "execution_count": null, "id": "1822499e", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "c26bf064", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:48.293465100Z", - "start_time": "2024-03-04T14:53:25.396221100Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_30119/3568740027.py:4: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " pre_data.rename(columns={'label_mortality_1month': 'label'}, inplace=True)\n" - ] - } - ], + "metadata": {}, + "outputs": [], "source": [ "# Load datasets\n", "finetune_dataset = FinetuneDataset(\n", @@ -236,9 +147,7 @@ "cell_type": "code", "execution_count": null, "id": "711ec104", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "data = pd.read_parquet(\n", @@ -256,15 +165,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "019ffd3a", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:48.293465100Z", - "start_time": "2024-03-04T14:53:48.277692100Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "train_loader = DataLoader(\n", @@ -275,71 +178,20 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "205463c5", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:48.434153700Z", - "start_time": "2024-03-04T14:53:48.277692100Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'concept_ids': tensor([ 5, 3, 10580, ..., 0, 0, 0]),\n", - " 'type_ids': tensor([1, 2, 7, ..., 0, 0, 0]),\n", - " 'ages': tensor([ 0, 76, 76, ..., 0, 0, 0]),\n", - " 'time_stamps': tensor([ 0, 5670, 5670, ..., 0, 0, 0]),\n", - " 'visit_orders': tensor([ 0, 2, 2, ..., 2049, 2049, 2049]),\n", - " 'visit_segments': tensor([0, 2, 2, ..., 0, 0, 0]),\n", - " 'labels': tensor(0),\n", - " 'attention_mask': tensor([1, 1, 1, ..., 0, 0, 0])}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "finetune_dataset[0]" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "eab1ec12", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-05T16:37:43.667550100Z", - "start_time": "2024-03-05T16:37:38.734456600Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:99: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.warmup = int(0.1 * grad_steps)\n", - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:100: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.decay = int(0.9 * grad_steps)\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "pretrained_model = BigBirdPretrain(\n", " args=args,\n", @@ -356,27 +208,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "e2de4cd6", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:53.187454700Z", - "start_time": "2024-03-04T14:53:53.187454700Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:283: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.warmup = int(0.1 * grad_steps)\n", - "/fs01/home/afallah/odyssey/odyssey/models/big_bird_cehr/model.py:284: FutureWarning: Calling int on a single element Series is deprecated and will raise a TypeError in the future. Use int(ser.iloc[0]) instead\n", - " self.decay = int(0.9 * grad_steps)\n" - ] - } - ], + "metadata": {}, + "outputs": [], "source": [ "model = BigBirdFinetune(\n", " args=args,\n", @@ -387,38 +222,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "aedab2ad", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:53.203166100Z", - "start_time": "2024-03-04T14:53:53.203166100Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'concept_ids': tensor([[ 5, 3, 10580, ..., 0, 0, 0],\n", - " [ 5, 3, 14013, ..., 0, 0, 0]]),\n", - " 'type_ids': tensor([[1, 2, 7, ..., 0, 0, 0],\n", - " [1, 2, 7, ..., 0, 0, 0]]),\n", - " 'ages': tensor([[ 0, 76, 76, ..., 0, 0, 0],\n", - " [ 0, 47, 47, ..., 0, 0, 0]]),\n", - " 'time_stamps': tensor([[ 0, 5670, 5670, ..., 0, 0, 0],\n", - " [ 0, 5857, 5857, ..., 0, 0, 0]]),\n", - " 'visit_orders': tensor([[ 0, 2, 2, ..., 2049, 2049, 2049],\n", - " [ 0, 2, 2, ..., 2049, 2049, 2049]]),\n", - " 'visit_segments': tensor([[0, 2, 2, ..., 0, 0, 0],\n", - " [0, 2, 2, ..., 0, 0, 0]])}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "batch = next(iter(train_loader))\n", "at = batch.pop(\"attention_mask\")\n", @@ -428,27 +235,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "1406a9b5", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:53:57.362579400Z", - "start_time": "2024-03-04T14:53:53.265211100Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([2, 2048, 20592])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "pretrained_model(\n", " inputs=tuple(batch.values()),\n", @@ -459,1065 +249,19 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "7bcb0f3d", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-04T14:57:51.517411500Z", - "start_time": "2024-03-04T14:57:46.724096800Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "SequenceClassifierOutput(loss=tensor(0.7985, grad_fn=), logits=tensor([[-0.0548, 0.3878],\n", - " [-0.0134, -0.0844]], grad_fn=), hidden_states=(tensor([[[-6.7883e-01, -2.1554e-01, -1.0080e+00, ..., -4.0627e-02,\n", - " -1.9321e-02, -2.8757e+00],\n", - " [-3.4303e-01, 1.1852e+00, -1.3026e-01, ..., 1.3585e+00,\n", - " -7.3872e-01, 2.7645e-01],\n", - " [-3.5166e-02, 1.1023e+00, 8.1303e-02, ..., 2.3114e-02,\n", - " 5.4960e-01, 2.8621e-01],\n", - " ...,\n", - " [-1.1241e+00, -3.8402e-01, -1.1593e+00, ..., -1.2536e-03,\n", - " 1.1522e-02, -3.2608e+00],\n", - " [-5.1603e-01, 4.8416e-01, -9.5265e-01, ..., -2.5094e+00,\n", - " 1.4269e-01, -3.4499e+00],\n", - " [-1.8325e-01, -1.2168e-01, 3.0352e-01, ..., -2.6697e+00,\n", - " 9.0409e-01, -2.5874e+00]],\n", - "\n", - " [[-6.7366e-01, -2.1113e-01, -1.0023e+00, ..., -3.6522e-02,\n", - " 1.4872e-02, 2.7641e-02],\n", - " [ 7.2113e-01, -4.1033e-02, -3.6217e-01, ..., 6.0504e-01,\n", - " -9.0939e-01, -5.1621e-01],\n", - " [-7.9435e-01, 3.1106e-01, -5.6777e-02, ..., 9.1777e-01,\n", - " -1.0380e+00, -1.3855e+00],\n", - " ...,\n", - " [ 4.0321e-03, -3.8975e-01, -1.1696e+00, ..., -1.6552e+00,\n", - " 7.9562e-01, 2.0766e-02],\n", - " [-5.3546e-01, 4.8907e-01, -9.8293e-01, ..., -2.5772e+00,\n", - " 1.3924e-01, -3.5412e+00],\n", - " [-1.5343e-01, -9.1049e-02, 3.3951e-01, ..., -2.6699e+00,\n", - " 9.4689e-01, -2.5867e+00]]], grad_fn=), tensor([[[-0.7612, 1.1788, 0.0670, ..., 0.1510, 0.1298, -1.9525],\n", - " [-0.1294, 0.7073, 0.4009, ..., 0.8259, 0.4033, -0.2180],\n", - " [ 0.0060, 1.0692, 1.1758, ..., 0.2298, 1.9952, -0.4547],\n", - " ...,\n", - " [-0.6428, -0.2628, -0.5125, ..., -0.0165, 1.8732, -2.3093],\n", - " [-0.7330, 0.3497, -0.1833, ..., -1.5858, 2.5036, -2.5915],\n", - " [-0.4628, 1.2452, 0.6572, ..., -1.4398, 0.6281, -2.0644]],\n", - "\n", - " [[-0.6763, 0.9164, -0.7162, ..., -0.3015, 2.1765, -0.3548],\n", - " [ 0.4091, -0.2961, -0.1489, ..., 0.3765, 0.3155, -1.0283],\n", - " [-0.7226, 0.1549, -0.4162, ..., 0.3072, 0.1819, -1.7718],\n", - " ...,\n", - " [ 0.2941, 0.8379, -0.4572, ..., -1.1039, 2.3471, -0.1551],\n", - " [-0.5228, 0.3502, -0.2576, ..., -1.6769, 2.4051, -2.8819],\n", - " [-0.5138, 1.1507, 0.4487, ..., -1.3191, 3.3422, -1.7425]]],\n", - " grad_fn=), tensor([[[-6.6177e-01, 7.5838e-01, 8.4508e-01, ..., 3.1616e-01,\n", - " -9.5201e-02, -1.9357e+00],\n", - " [-5.4649e-01, 1.1234e+00, 7.9796e-01, ..., 8.9291e-01,\n", - " 6.7732e-01, -2.2775e-02],\n", - " [ 8.7776e-03, 1.0608e+00, 1.6783e+00, ..., 7.2717e-01,\n", - " 1.6662e+00, -4.8948e-01],\n", - " ...,\n", - " [-2.4893e-01, 9.1186e-01, 4.3222e-01, ..., 2.7560e-01,\n", - " 2.0665e+00, -2.3981e+00],\n", - " [-8.3648e-01, 2.4201e-01, 8.6920e-01, ..., -6.1769e-01,\n", - " 2.3556e+00, -2.6167e+00],\n", - " [-6.0143e-01, 1.7220e+00, 1.3712e+00, ..., -2.2286e-01,\n", - " 1.4262e+00, -2.3164e+00]],\n", - "\n", - " [[-5.2576e-01, 1.4678e+00, -1.0744e-03, ..., -2.9040e-01,\n", - " 2.1817e+00, -7.4288e-01],\n", - " [ 2.6891e-01, -3.1736e-02, 9.0947e-01, ..., 5.6953e-01,\n", - " 8.5472e-01, -8.2515e-01],\n", - " [-6.1062e-01, 1.3617e-01, -3.9544e-01, ..., 3.7781e-02,\n", - " 1.0965e+00, -1.6974e+00],\n", - " ...,\n", - " [ 1.7578e-01, 1.5607e+00, 4.1353e-01, ..., -4.9145e-01,\n", - " 1.5555e+00, -1.1926e-01],\n", - " [-5.9141e-01, 1.3380e+00, 8.2107e-01, ..., -5.8079e-01,\n", - " 2.2684e+00, -2.0261e+00],\n", - " [-5.6457e-01, 1.7507e+00, 1.0470e+00, ..., -1.4447e-01,\n", - " 3.0054e+00, -2.3032e+00]]], grad_fn=), tensor([[[-0.8392, 0.7790, 0.5453, ..., 1.0194, 0.9533, -1.6746],\n", - " [-0.2692, 0.7316, 1.4549, ..., 1.3333, 1.4484, -0.8517],\n", - " [ 0.6038, 0.5935, 1.5841, ..., 0.8122, 1.8548, -1.4039],\n", - " ...,\n", - " [-0.2305, 0.8916, 0.7803, ..., 0.8734, 2.1020, -2.0518],\n", - " [-1.0039, 0.3233, 0.8694, ..., -0.0389, 2.4537, -1.9296],\n", - " [-0.7582, 1.4646, 1.0067, ..., 0.2241, 1.8502, -2.1887]],\n", - "\n", - " [[-1.2226, 0.9212, -0.4557, ..., 0.7558, 2.2563, -0.5759],\n", - " [ 0.1584, -0.2660, 1.6992, ..., 1.1529, 1.2050, -1.3101],\n", - " [-1.0150, -0.3003, -0.1375, ..., 0.3245, 1.5134, -1.4331],\n", - " ...,\n", - " [-0.0692, 1.2825, 0.7375, ..., 0.1073, 1.1074, -0.4870],\n", - " [-0.7067, 1.0616, 0.7769, ..., -0.0338, 2.2439, -1.8917],\n", - " [-0.8515, 1.4562, 0.7459, ..., 0.3793, 2.6386, -1.6685]]],\n", - " grad_fn=), tensor([[[-0.9823, 0.2108, 0.4440, ..., 2.0734, 0.3965, -1.7823],\n", - " [-0.1377, 0.8754, 1.8478, ..., 1.7912, 1.0556, -0.7279],\n", - " [ 0.1172, 0.3067, 2.4020, ..., 1.4624, 1.1320, -1.9218],\n", - " ...,\n", - " [-0.2582, 0.5947, 1.5271, ..., 1.0780, 1.4205, -1.6517],\n", - " [-0.9463, 0.3378, 1.5753, ..., 0.5881, 1.7042, -1.8454],\n", - " [-0.8436, 0.8822, 1.5665, ..., 0.7536, 1.0132, -1.5555]],\n", - "\n", - " [[-1.6627, 0.5257, 0.4442, ..., 1.5420, 1.2081, -0.5087],\n", - " [-0.0228, -0.0685, 1.8292, ..., 1.5382, 1.4164, -1.1278],\n", - " [-1.2641, -0.0808, 0.3799, ..., -0.0749, 1.6157, -0.8058],\n", - " ...,\n", - " [-0.1188, 0.6226, 1.7203, ..., 0.0791, 0.7161, -0.5938],\n", - " [-0.5220, 0.7746, 0.5732, ..., -0.0152, 1.3219, -1.7901],\n", - " [-0.6004, 0.8353, 1.3275, ..., 0.6180, 1.6479, -1.3394]]],\n", - " grad_fn=), tensor([[[-2.5259e-01, -1.2459e-01, 7.5976e-03, ..., 1.9570e+00,\n", - " -4.6990e-01, -1.4634e+00],\n", - " [ 1.0143e+00, 3.9836e-01, 1.3978e+00, ..., 2.1796e+00,\n", - " -1.4753e-01, -9.7475e-01],\n", - " [ 1.1543e+00, -1.0080e-03, 1.9922e+00, ..., 1.3462e+00,\n", - " 8.2855e-02, -1.7478e+00],\n", - " ...,\n", - " [-3.1573e-01, 4.9334e-01, 1.4406e+00, ..., 2.3396e-01,\n", - " 1.2609e+00, -1.1076e+00],\n", - " [-7.3841e-01, 2.8752e-01, 1.5985e+00, ..., -5.4855e-02,\n", - " 1.6283e+00, -1.4356e+00],\n", - " [-5.5749e-01, 8.5644e-01, 1.5186e+00, ..., -3.0065e-01,\n", - " 1.1832e+00, -1.3197e+00]],\n", - "\n", - " [[-1.0105e+00, 3.1137e-01, -6.4281e-01, ..., 1.2526e+00,\n", - " 8.7780e-01, -8.6638e-01],\n", - " [ 1.6084e+00, -1.5660e-01, 1.3290e+00, ..., 1.2241e+00,\n", - " 5.6264e-01, -1.5534e+00],\n", - " [ 4.1183e-01, -6.7924e-01, 2.9941e-01, ..., 2.4660e-01,\n", - " 1.3763e+00, -1.4622e+00],\n", - " ...,\n", - " [-6.8823e-02, 4.6075e-01, 1.6617e+00, ..., -5.6571e-01,\n", - " 6.4098e-01, -6.6053e-01],\n", - " [-4.9068e-01, 5.9720e-01, 7.4771e-01, ..., -8.3096e-01,\n", - " 1.1956e+00, -1.3331e+00],\n", - " [-4.5931e-01, 7.1495e-01, 1.2672e+00, ..., 5.1930e-01,\n", - " 1.2923e+00, -1.1016e+00]]], grad_fn=), tensor([[[ 0.3742, -0.0764, 0.0945, ..., 1.4158, -0.8588, -1.6218],\n", - " [ 1.4871, 0.1186, 1.1610, ..., 2.0626, -0.0506, -1.5310],\n", - " [ 1.6674, 0.4859, 2.0500, ..., 1.2819, -0.1244, -1.3927],\n", - " ...,\n", - " [ 0.0864, 0.6082, 1.3505, ..., -0.2613, 0.5019, -1.0052],\n", - " [-0.3482, 0.2363, 1.3896, ..., -0.4732, 0.9287, -1.6733],\n", - " [-0.1332, 0.8451, 1.4578, ..., -0.8543, 0.4736, -1.3068]],\n", - "\n", - " [[-0.6323, 0.5033, -0.6140, ..., 0.9162, -0.0629, -1.0184],\n", - " [ 1.8770, 0.1320, 1.1057, ..., 1.4696, 0.2647, -2.0210],\n", - " [ 0.4418, 0.0900, 0.1998, ..., 0.7580, 0.5991, -2.0740],\n", - " ...,\n", - " [-0.0496, 0.2864, 1.5863, ..., -1.1051, 0.0382, -0.5217],\n", - " [-0.0280, 0.7463, 0.9009, ..., -1.3713, 0.5620, -1.2647],\n", - " [-0.1137, 0.9139, 1.2550, ..., 0.4390, 0.4861, -0.8746]]],\n", - " grad_fn=)), attentions=(tensor([[[[3.3829e-01, 4.2168e-04, 6.0371e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9615e-03, 5.6922e-03, 3.4314e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.8037e-03, 5.1372e-03, 1.1139e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.4716e-02, 2.1618e-03, 2.0521e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0077e-02, 1.5925e-03, 2.5802e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1897e-02, 1.3985e-03, 1.9099e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.4613e-05, 4.7240e-03, 2.7755e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.3412e-04, 1.9313e-03, 1.9358e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2866e-04, 3.8389e-03, 3.1062e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.7880e-04, 5.5953e-03, 3.5820e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.8394e-05, 5.3725e-03, 3.2042e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4452e-05, 6.1726e-03, 3.5660e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.9910e-04, 3.0474e-03, 2.5833e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7465e-03, 1.5840e-03, 2.2462e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4932e-03, 2.9983e-03, 6.0509e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [5.8728e-03, 2.5286e-03, 2.5139e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.0072e-04, 3.7529e-03, 3.5507e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.4507e-04, 3.4646e-03, 3.0753e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.2004e-04, 7.4413e-04, 4.8160e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9437e-03, 1.2724e-03, 1.6742e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4512e-03, 1.5592e-03, 2.7156e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.2059e-04, 9.2442e-04, 3.1271e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3537e-04, 7.5489e-04, 4.9048e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5703e-04, 1.3878e-03, 4.3348e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.0847e-03, 9.9365e-04, 1.4654e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9991e-03, 2.5298e-03, 2.0633e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3427e-03, 2.6584e-03, 3.4574e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6549e-03, 2.0082e-03, 2.8673e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1673e-03, 1.2818e-03, 2.1908e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2479e-03, 1.2427e-03, 1.6143e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.5453e-03, 2.4796e-03, 2.0932e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1818e-03, 1.5144e-03, 1.8244e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.5179e-03, 2.5752e-03, 1.5294e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [5.7903e-03, 1.8382e-03, 1.4779e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7250e-03, 1.9423e-03, 1.2365e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.5668e-03, 1.9948e-03, 1.6985e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]],\n", - "\n", - "\n", - " [[[4.8522e-01, 2.6012e-03, 5.0588e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4163e-02, 1.0012e-02, 9.6521e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0227e-02, 9.3909e-03, 1.6390e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.3564e-02, 6.2564e-03, 8.9229e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.9858e-02, 6.9418e-03, 1.0739e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.8997e-02, 6.0291e-03, 9.4280e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.4369e-05, 4.5977e-03, 7.7826e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.6617e-04, 5.1484e-03, 6.8859e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9042e-03, 5.9801e-03, 5.3551e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [5.8818e-03, 7.5334e-03, 6.8311e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5379e-03, 4.5522e-03, 7.9804e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7689e-05, 3.3695e-03, 7.6476e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.3898e-03, 9.8130e-03, 7.7340e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.2367e-02, 1.0372e-02, 1.0222e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5844e-02, 1.5915e-02, 1.5106e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.5293e-03, 1.5424e-02, 9.2990e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7154e-03, 2.0451e-02, 1.3884e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1837e-03, 1.4825e-02, 1.0105e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[8.7333e-04, 7.5309e-03, 1.4210e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7119e-03, 8.9821e-03, 1.5552e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.8669e-03, 8.8078e-03, 1.3934e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.1273e-03, 7.9804e-03, 1.2161e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7193e-03, 8.8673e-03, 1.1518e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.3908e-04, 7.0001e-03, 1.3040e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.4784e-03, 5.6126e-03, 7.5390e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4794e-03, 5.2695e-03, 6.8683e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0646e-03, 5.5436e-03, 7.5504e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.0913e-03, 8.1178e-03, 7.6516e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.0621e-03, 8.1014e-03, 8.8541e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9311e-03, 6.0918e-03, 6.3840e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.5190e-03, 3.7157e-03, 4.6686e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.1103e-03, 7.2042e-03, 6.3460e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.6153e-03, 1.0473e-02, 6.4398e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0222e-02, 6.3617e-03, 4.7274e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.1297e-03, 6.4029e-03, 4.7556e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.0694e-03, 6.7958e-03, 5.2921e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], grad_fn=), tensor([[[[1.1852e-03, 1.2116e-03, 4.4657e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.2031e-03, 3.4566e-03, 3.5255e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.3378e-03, 2.9312e-03, 3.0110e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.9520e-03, 2.1999e-03, 3.6661e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3657e-03, 1.5178e-03, 6.5193e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3469e-03, 2.0453e-03, 4.5768e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.8409e-05, 2.6280e-03, 1.4181e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5274e-04, 4.4478e-03, 3.1983e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.5926e-03, 5.5057e-03, 3.5726e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.3798e-04, 4.8291e-03, 2.1062e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0653e-04, 3.5371e-03, 1.7556e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3074e-04, 3.9381e-03, 1.3827e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.0259e-03, 2.5137e-03, 4.1686e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7744e-03, 6.4283e-03, 1.1902e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.9855e-03, 5.6803e-03, 2.3984e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.9519e-03, 1.5009e-03, 1.2247e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9123e-03, 1.6541e-03, 1.6026e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8178e-03, 1.0684e-03, 8.0199e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.8556e-03, 6.1808e-03, 3.2089e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1088e-03, 7.3328e-03, 3.5654e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7393e-03, 1.5942e-02, 7.2926e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.7402e-04, 8.6555e-03, 3.2202e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.1968e-04, 8.2633e-03, 4.7004e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4571e-03, 8.0842e-03, 3.2649e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.6902e-03, 2.7200e-03, 1.9858e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0243e-03, 8.6233e-03, 2.1934e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0941e-03, 1.1516e-02, 2.6320e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.9088e-03, 3.0460e-03, 2.8220e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.1548e-03, 2.1842e-03, 2.7460e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.4365e-03, 2.1235e-03, 2.8106e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.1677e-03, 1.0207e-03, 2.1433e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0661e-03, 1.7767e-03, 2.3279e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.4730e-03, 3.5249e-03, 7.5658e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.1904e-03, 6.6556e-04, 1.2853e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7925e-03, 1.5353e-03, 2.5354e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0067e-03, 6.5366e-04, 9.6887e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]],\n", - "\n", - "\n", - " [[[5.1334e-03, 4.1015e-03, 1.1227e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9587e-02, 2.8289e-02, 1.0980e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1242e-02, 2.1415e-02, 9.9579e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.6650e-03, 5.9826e-03, 1.0389e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.1852e-03, 5.9235e-03, 1.3426e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.3821e-03, 6.0841e-03, 1.4991e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.3487e-04, 4.2364e-03, 5.4961e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.2315e-04, 7.0932e-03, 6.9453e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1437e-02, 1.4434e-02, 1.3050e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.2482e-04, 7.8274e-03, 5.8569e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.4739e-04, 6.8490e-03, 5.1440e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.1406e-04, 9.0954e-03, 4.4066e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.2141e-02, 5.1091e-03, 4.3978e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1660e-03, 8.4786e-03, 1.0966e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0776e-02, 8.4128e-03, 1.4434e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.4194e-03, 8.1452e-03, 4.5854e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0779e-02, 7.6571e-03, 6.0113e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.7561e-03, 8.6070e-03, 3.4526e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[2.7547e-02, 3.6981e-02, 7.7727e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3315e-02, 5.5422e-02, 7.5489e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1363e-02, 1.3585e-02, 6.3071e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.2465e-03, 3.9692e-02, 1.2010e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.8985e-03, 7.9480e-02, 1.5213e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.1275e-03, 8.3321e-02, 1.2849e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.0286e-02, 1.7433e-02, 1.0044e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7478e-03, 3.2198e-02, 1.2874e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.7016e-03, 1.2031e-02, 1.0720e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6077e-02, 3.2117e-02, 1.2547e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9592e-02, 2.0783e-02, 1.0426e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.8511e-02, 2.5143e-02, 1.3227e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[7.5246e-03, 2.9466e-03, 3.5927e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.2404e-03, 5.1474e-03, 5.5319e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0508e-02, 1.3039e-02, 9.9883e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0351e-02, 3.1721e-03, 4.4100e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.7617e-03, 3.7442e-03, 5.7102e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2872e-02, 3.0914e-03, 2.8024e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], grad_fn=), tensor([[[[5.4818e-04, 9.4356e-04, 1.8028e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.2519e-04, 1.3145e-03, 5.9805e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.9195e-04, 1.1742e-03, 1.0372e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [6.2802e-05, 6.8331e-04, 4.6733e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9824e-04, 5.5174e-04, 6.6800e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0214e-04, 6.4142e-04, 6.3604e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[6.1718e-04, 3.6397e-03, 2.5932e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8767e-05, 2.0277e-03, 4.7040e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6359e-05, 3.3142e-03, 5.9716e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.1284e-04, 1.5191e-03, 1.5340e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.7136e-04, 2.2989e-03, 2.0590e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4346e-04, 1.6801e-03, 1.1624e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.9753e-03, 1.5102e-03, 5.5539e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.5413e-04, 1.4099e-02, 3.5474e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7997e-04, 5.3596e-03, 4.8483e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.7207e-03, 3.0669e-02, 3.0692e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.4813e-04, 3.1476e-03, 2.4708e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.3242e-04, 7.6870e-03, 1.6922e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[4.6202e-04, 4.3252e-03, 3.9071e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.8002e-05, 1.2901e-02, 3.8641e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.7992e-05, 8.3308e-03, 7.6479e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.1665e-04, 5.1651e-03, 3.9104e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.6917e-04, 3.2589e-03, 3.2665e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0006e-04, 4.2733e-03, 3.3829e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.8080e-04, 6.6270e-03, 4.6011e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.1309e-04, 3.6184e-03, 4.5401e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9300e-04, 3.5974e-03, 3.0364e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.1472e-04, 8.1332e-03, 2.2159e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4566e-04, 2.5894e-03, 1.3540e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8549e-04, 4.0805e-03, 8.5880e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.5426e-03, 1.9465e-03, 2.9837e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2093e-03, 2.4648e-03, 3.1128e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3917e-03, 1.3005e-03, 2.8057e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.1829e-03, 1.8707e-03, 2.0393e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8815e-03, 1.9994e-03, 2.2172e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6373e-03, 1.6550e-03, 2.0867e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]],\n", - "\n", - "\n", - " [[[2.7360e-03, 1.1008e-02, 7.3260e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.6371e-03, 6.1147e-02, 1.5480e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.5041e-03, 1.9268e-03, 1.7407e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [8.7758e-05, 2.5530e-03, 5.6661e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.0306e-04, 5.8363e-03, 1.3576e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1090e-04, 1.0685e-02, 2.1807e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.1591e-03, 1.1866e-02, 4.9158e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.8648e-04, 8.8206e-03, 4.9711e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1607e-03, 6.6525e-03, 8.9679e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.3172e-03, 7.2261e-03, 4.0674e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4690e-03, 1.0568e-02, 6.5783e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7593e-03, 5.1639e-03, 3.0872e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[9.3591e-04, 3.0583e-04, 1.2086e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2840e-03, 2.5355e-02, 4.0486e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0034e-03, 1.3239e-03, 7.2349e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.7158e-03, 1.6269e-02, 2.5834e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4881e-03, 2.4591e-02, 5.9714e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.8160e-03, 2.2059e-02, 4.7236e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[1.6208e-03, 7.6657e-03, 6.7118e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5970e-04, 1.0342e-02, 3.9979e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.8039e-04, 9.2103e-03, 6.2737e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.0275e-04, 1.3967e-02, 5.9975e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.4287e-03, 1.0881e-02, 6.9529e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2096e-03, 1.1784e-02, 7.3909e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.5804e-04, 4.0411e-03, 3.3539e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.1780e-03, 7.1733e-02, 3.8132e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7373e-04, 1.1137e-03, 5.2212e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.4057e-04, 2.1332e-02, 2.0725e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.5246e-04, 2.6167e-02, 4.4173e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.7590e-04, 1.4341e-02, 2.3822e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[5.1459e-03, 6.4601e-03, 8.7169e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.0140e-03, 1.0282e-02, 1.8618e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.5989e-03, 5.0204e-03, 1.4960e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.1973e-03, 6.2178e-03, 8.0227e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.2183e-03, 8.3229e-03, 8.4244e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7811e-03, 6.4020e-03, 1.0923e-02, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], grad_fn=), tensor([[[[2.9685e-03, 2.4337e-03, 1.2576e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2044e-03, 8.9882e-03, 2.8169e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.0168e-03, 6.2733e-03, 2.1396e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.6333e-03, 6.5531e-03, 2.7376e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.9913e-04, 5.2101e-03, 2.6853e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [6.4545e-04, 4.2812e-03, 1.0562e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[2.8730e-03, 2.3672e-03, 3.3017e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3382e-04, 1.6171e-03, 1.6500e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.9031e-04, 2.0492e-03, 2.3732e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [5.8339e-04, 3.9774e-03, 3.6571e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.9300e-04, 2.7660e-03, 2.7645e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.7934e-04, 1.7628e-03, 1.3876e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.5549e-03, 1.0770e-03, 5.7588e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.4111e-05, 2.6560e-03, 5.7361e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0032e-03, 6.6076e-04, 1.5630e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [4.5366e-04, 5.6356e-03, 1.8294e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6178e-04, 2.7165e-03, 1.4668e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0544e-04, 1.7698e-03, 9.5275e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[3.1439e-04, 1.4112e-04, 6.0595e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2998e-04, 2.5551e-04, 5.4382e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.2105e-04, 3.3150e-04, 1.0713e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.0085e-04, 7.2958e-04, 1.1065e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.7959e-05, 2.9731e-04, 4.5974e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.3942e-05, 1.2093e-04, 1.7051e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[9.1484e-03, 9.0738e-04, 6.2215e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.9197e-02, 1.0122e-02, 1.6618e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.8152e-03, 9.8951e-04, 7.2292e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.2452e-02, 1.9161e-02, 3.4519e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.3776e-03, 1.0579e-02, 2.9252e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0429e-03, 2.6139e-03, 5.6355e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[8.3608e-04, 6.4050e-04, 9.5779e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1852e-04, 5.0046e-04, 2.3925e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.1351e-04, 1.0930e-03, 1.9739e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [3.4037e-04, 1.3694e-03, 1.3412e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.6337e-04, 1.6551e-03, 2.9647e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [8.5230e-05, 3.4489e-04, 7.0684e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]],\n", - "\n", - "\n", - " [[[3.4924e-03, 2.4305e-03, 3.8081e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3378e-03, 2.5938e-02, 4.6432e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.3373e-03, 1.1847e-03, 6.4010e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.9032e-03, 1.5456e-02, 5.4833e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [4.8012e-03, 2.9131e-02, 7.6610e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.3769e-03, 2.2525e-02, 4.8307e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.5474e-03, 1.6992e-03, 1.0799e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0776e-03, 2.7245e-02, 1.2739e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.6883e-04, 4.3499e-05, 9.7167e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.1314e-04, 2.1016e-03, 2.7716e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.0334e-03, 2.6151e-02, 1.9928e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.2660e-04, 7.2961e-03, 4.3441e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[7.6996e-04, 1.2438e-04, 4.9751e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.5833e-03, 3.7775e-02, 1.4243e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.2042e-04, 1.7617e-05, 7.9105e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [1.0450e-04, 5.5392e-04, 1.6018e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.1257e-04, 6.2165e-03, 9.1572e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [9.8000e-05, 7.5413e-03, 8.7500e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " ...,\n", - "\n", - " [[4.4917e-03, 3.1300e-04, 1.5339e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4840e-03, 7.8944e-03, 1.5286e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.2337e-03, 2.3978e-04, 3.4733e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [2.4593e-04, 4.8222e-04, 5.4714e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.1030e-03, 4.1702e-03, 1.6100e-03, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0183e-04, 9.7633e-04, 6.0602e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[1.5175e-02, 5.4381e-04, 8.5634e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3616e-01, 9.7432e-02, 6.5399e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [2.5971e-03, 1.2464e-04, 3.3438e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [9.4937e-03, 1.0469e-02, 6.7031e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [5.9924e-02, 1.0426e-01, 2.8601e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.3472e-02, 2.9242e-02, 1.4018e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]],\n", - "\n", - " [[4.2284e-04, 8.2281e-05, 4.1306e-05, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [1.0693e-03, 4.0504e-03, 1.4109e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [7.1268e-04, 1.7843e-04, 3.4107e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " ...,\n", - " [5.4440e-04, 3.4202e-03, 3.4268e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.7159e-03, 2.0749e-02, 7.3614e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00],\n", - " [3.4170e-04, 5.8382e-03, 4.2085e-04, ..., 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00]]]], grad_fn=), tensor([[[[0.0051, 0.0019, 0.0024, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0016, 0.0019, 0.0016, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0071, 0.0030, 0.0070, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0013, 0.0025, 0.0026, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0018, 0.0021, 0.0031, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0016, 0.0024, 0.0026, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0011, 0.0004, 0.0008, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0012, 0.0008, 0.0021, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0020, 0.0010, 0.0021, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0008, 0.0009, 0.0012, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0007, 0.0008, 0.0012, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0006, 0.0008, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0044, 0.0045, 0.0030, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0056, 0.0008, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0014, 0.0015, 0.0020, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0008, 0.0048, 0.0040, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0034, 0.0024, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0035, 0.0023, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " ...,\n", - "\n", - " [[0.0042, 0.0021, 0.0028, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0025, 0.0020, 0.0016, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0023, 0.0010, 0.0020, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0025, 0.0037, 0.0031, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0022, 0.0041, 0.0034, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0024, 0.0032, 0.0026, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0018, 0.0008, 0.0026, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0016, 0.0029, 0.0035, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0029, 0.0017, 0.0024, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0014, 0.0021, 0.0030, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0014, 0.0020, 0.0044, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0011, 0.0013, 0.0031, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0021, 0.0014, 0.0018, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0003, 0.0020, 0.0024, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0017, 0.0006, 0.0026, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0014, 0.0023, 0.0030, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0010, 0.0038, 0.0040, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0020, 0.0022, ..., 0.0000, 0.0000, 0.0000]]],\n", - "\n", - "\n", - " [[[0.0060, 0.0031, 0.0023, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0174, 0.0575, 0.0028, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0162, 0.0046, 0.0067, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0017, 0.0042, 0.0015, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0032, 0.0089, 0.0016, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0058, 0.0188, 0.0037, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0007, 0.0002, 0.0015, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0051, 0.0073, 0.0043, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0009, 0.0004, 0.0068, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0016, 0.0027, 0.0032, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0034, 0.0085, 0.0040, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0027, 0.0059, 0.0034, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0062, 0.0067, 0.0044, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0027, 0.0439, 0.0030, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0010, 0.0003, 0.0043, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0018, 0.0226, 0.0024, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0020, 0.0638, 0.0022, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0020, 0.0574, 0.0043, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " ...,\n", - "\n", - " [[0.0111, 0.0061, 0.0044, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0087, 0.0155, 0.0064, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0154, 0.0015, 0.0056, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0035, 0.0130, 0.0037, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0051, 0.0196, 0.0044, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0086, 0.0283, 0.0065, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0028, 0.0008, 0.0034, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0041, 0.0220, 0.0039, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0051, 0.0014, 0.0082, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0028, 0.0039, 0.0038, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0050, 0.0105, 0.0058, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0058, 0.0094, 0.0090, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0010, 0.0002, 0.0009, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0039, 0.0191, 0.0033, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0015, 0.0003, 0.0028, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0017, 0.0042, 0.0020, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0036, 0.0164, 0.0032, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0046, 0.0260, 0.0057, ..., 0.0000, 0.0000, 0.0000]]]],\n", - " grad_fn=), tensor([[[[0.0024, 0.0020, 0.0015, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0029, 0.0049, 0.0017, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0020, 0.0023, 0.0021, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0027, 0.0027, 0.0020, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0031, 0.0026, 0.0023, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0025, 0.0032, 0.0023, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0018, 0.0039, 0.0018, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0007, 0.0041, 0.0009, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0005, 0.0037, 0.0042, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0021, 0.0024, 0.0025, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0017, 0.0027, 0.0017, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0018, 0.0032, 0.0014, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0021, 0.0013, 0.0016, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0016, 0.0017, 0.0021, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0009, 0.0011, 0.0034, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0024, 0.0020, 0.0011, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0022, 0.0016, 0.0010, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0026, 0.0016, 0.0011, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " ...,\n", - "\n", - " [[0.0029, 0.0020, 0.0023, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0017, 0.0018, 0.0014, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0012, 0.0019, 0.0039, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0031, 0.0016, 0.0030, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0029, 0.0013, 0.0026, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0032, 0.0012, 0.0031, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0022, 0.0012, 0.0009, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0021, 0.0023, 0.0014, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0019, 0.0018, 0.0028, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0016, 0.0016, 0.0009, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0013, 0.0015, 0.0007, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0012, 0.0010, 0.0006, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0043, 0.0019, 0.0032, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0030, 0.0019, 0.0013, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0026, 0.0012, 0.0035, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0036, 0.0018, 0.0042, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0034, 0.0022, 0.0039, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0031, 0.0020, 0.0040, ..., 0.0000, 0.0000, 0.0000]]],\n", - "\n", - "\n", - " [[[0.0044, 0.0017, 0.0057, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0041, 0.0112, 0.0066, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0027, 0.0025, 0.0045, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0089, 0.0058, 0.0067, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0106, 0.0065, 0.0056, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0081, 0.0080, 0.0068, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0070, 0.0070, 0.0055, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0051, 0.0081, 0.0042, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0027, 0.0077, 0.0058, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0094, 0.0098, 0.0089, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0080, 0.0114, 0.0084, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0087, 0.0123, 0.0091, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0091, 0.0039, 0.0100, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0050, 0.0049, 0.0068, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0071, 0.0029, 0.0116, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0095, 0.0035, 0.0080, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0091, 0.0035, 0.0095, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0082, 0.0036, 0.0109, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " ...,\n", - "\n", - " [[0.0091, 0.0092, 0.0085, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0038, 0.0069, 0.0058, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0033, 0.0074, 0.0093, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0126, 0.0100, 0.0106, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0135, 0.0112, 0.0096, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0134, 0.0111, 0.0090, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0062, 0.0013, 0.0062, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0059, 0.0064, 0.0056, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0106, 0.0041, 0.0071, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0037, 0.0027, 0.0070, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0038, 0.0023, 0.0070, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0023, 0.0024, 0.0071, ..., 0.0000, 0.0000, 0.0000]],\n", - "\n", - " [[0.0112, 0.0078, 0.0074, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0066, 0.0087, 0.0053, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0046, 0.0054, 0.0072, ..., 0.0000, 0.0000, 0.0000],\n", - " ...,\n", - " [0.0085, 0.0106, 0.0064, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0121, 0.0081, 0.0071, ..., 0.0000, 0.0000, 0.0000],\n", - " [0.0088, 0.0065, 0.0063, ..., 0.0000, 0.0000, 0.0000]]]],\n", - " grad_fn=)))" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "model(inputs=tuple(batch.values()), attention_mask=at, labels=labels, return_dict=True)" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "8c0ceac3", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-01T15:14:52.803317400Z", - "start_time": "2024-03-01T15:14:52.789656900Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "def mask_tokens(self, sequence: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:\n", @@ -1554,27 +298,10 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "a43549ee", - "metadata": { - "ExecuteTime": { - "end_time": "2024-03-01T15:21:55.944148400Z", - "start_time": "2024-03-01T15:21:55.921896700Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "9" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "len(set(train_dataset[0][\"type_ids\"].tolist()))" ] @@ -1583,12 +310,7 @@ "cell_type": "code", "execution_count": null, "id": "73756c75059737d3", - "metadata": { - "ExecuteTime": { - "start_time": "2024-02-26T15:46:43.081570800Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "patients = pd.read_parquet(NEW_DATA_PATH)\n", @@ -1597,15 +319,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "acdfaf82", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-26T15:40:30.829556400Z", - "start_time": "2024-02-26T15:40:30.745437200Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer = HuggingFaceConceptTokenizer(data_dir=DATA_ROOT)\n", @@ -1621,15 +337,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0eaf26ec", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-26T15:40:30.844276Z", - "start_time": "2024-02-26T15:40:30.836556300Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "e1 = (\n", @@ -1640,39 +350,10 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": null, "id": "1a5cfe30", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-26T15:46:35.854499500Z", - "start_time": "2024-02-26T15:46:35.589931400Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'input_ids': tensor([[ 5, 0, 0, ..., 0, 0, 0],\n", - " [ 3, 0, 0, ..., 0, 0, 0],\n", - " [12809, 0, 0, ..., 0, 0, 0],\n", - " ...,\n", - " [ 1352, 0, 0, ..., 0, 0, 0],\n", - " [ 4, 0, 0, ..., 0, 0, 0],\n", - " [ 6, 0, 0, ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 0, 0, ..., 0, 0, 0],\n", - " [1, 0, 0, ..., 0, 0, 0],\n", - " [1, 0, 0, ..., 0, 0, 0],\n", - " ...,\n", - " [1, 0, 0, ..., 0, 0, 0],\n", - " [1, 0, 0, ..., 0, 0, 0],\n", - " [1, 0, 0, ..., 0, 0, 0]])}" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "tokenizer(patients[\"event_tokens_2048\"].iloc[0])" ] @@ -1681,9 +362,7 @@ "cell_type": "code", "execution_count": null, "id": "6f181cd517e54378", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "patients = patients[patients[\"event_tokens_2048\"].notnull()]\n", @@ -1703,9 +382,7 @@ "cell_type": "code", "execution_count": null, "id": "a024b90f7a8241b4", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer.decode([[3]])" @@ -1715,9 +392,7 @@ "cell_type": "code", "execution_count": null, "id": "617ef0182b3a60b7", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "patients.iloc[1]" @@ -1727,9 +402,7 @@ "cell_type": "code", "execution_count": null, "id": "147baf6cd3558cb5", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "train_dataset[0]" @@ -1739,9 +412,7 @@ "cell_type": "code", "execution_count": null, "id": "77a76df1a0358049", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "print(train_dataset[0][\"attention_mask\"])" @@ -1751,9 +422,7 @@ "cell_type": "code", "execution_count": null, "id": "65076f3cfce1754a", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer.get_mask_token_id()" @@ -1763,9 +432,7 @@ "cell_type": "code", "execution_count": null, "id": "cc34f08b3313b37b", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "print(list(train_dataset[110][\"concept_ids\"]).count(20569))" @@ -1775,9 +442,7 @@ "cell_type": "code", "execution_count": null, "id": "a73b9cef89b98c7e", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "print(len(train_dataset[110][\"concept_ids\"]))" @@ -1787,9 +452,7 @@ "cell_type": "code", "execution_count": null, "id": "1765143ee1fbb703", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer.get_pad_token_id()" @@ -1799,9 +462,7 @@ "cell_type": "code", "execution_count": null, "id": "bbb70c831ad4de96", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "tokenizer.encode([\"[PAD]\"])" @@ -1811,9 +472,7 @@ "cell_type": "code", "execution_count": null, "id": "93db816c5601357", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "patients.iloc[0][\"event_tokens_2048\"]" @@ -1823,9 +482,7 @@ "cell_type": "code", "execution_count": null, "id": "d7af63bd", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "ROOT = \"/fs01/home/afallah/odyssey/odyssey\"\n", diff --git a/odyssey/models/cehr_big_bird/tokenizer.ipynb b/odyssey/models/cehr_big_bird/tokenizer.ipynb index c611b76..bc3686d 100644 --- a/odyssey/models/cehr_big_bird/tokenizer.ipynb +++ b/odyssey/models/cehr_big_bird/tokenizer.ipynb @@ -2,31 +2,10 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "initial_id", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-13T22:43:09.964122800Z", - "start_time": "2024-02-13T22:42:40.345121300Z" - }, - "collapsed": true - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-02-13 17:42:53.460655: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", - "2024-02-13 17:42:53.911746: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-02-13 17:42:53.911900: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-02-13 17:42:53.971676: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-02-13 17:42:54.125474: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n", - "2024-02-13 17:42:54.134630: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-02-13 17:42:58.317896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" - ] - } - ], + "metadata": {}, + "outputs": [], "source": [ "import os\n", "\n", @@ -63,15 +42,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "eebfff5979c4709f", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-13T22:43:09.986121100Z", - "start_time": "2024-02-13T22:43:09.972119200Z" - }, - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "class config:\n", @@ -89,359 +62,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "1f188c4b86093192", - "metadata": { - "ExecuteTime": { - "end_time": "2024-02-13T22:43:58.611200300Z", - "start_time": "2024-02-13T22:43:09.979121800Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
patient_idnum_visitsdeceaseddeath_after_startdeath_after_endlengthtoken_lengthevent_tokens_2048type_tokens_2048age_tokens_2048time_tokens_2048visit_tokens_2048position_tokens_2048label
0f8f3289a-057f-5fcc-a714-5f6109ca16c420NaNNaN14[[CLS], [VS], 8938, [VE], [PAD], [PAD], [PAD],...[1, 2, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 8262, 8262, 8262, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 2, 2, 2, 2049, 2049, 2049, 2049, 2049, 204...0
19b62c9f4-3fdc-5020-82b5-ae5b8292445a40NaNNaN4352[[CLS], [VS], 7569, 66689036430, 00904224461, ...[1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 4, ...[0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28...[0, 5963, 5963, 5963, 5963, 5963, 5963, 5963, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
22ca522eb-dd89-5f79-8155-9599ea46b0b221244.0242.05154[[CLS], [VS], 00904629261, 00904642281, 009046...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86...[0, 8016, 8016, 8016, 8016, 8016, 8016, 8016, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
402adf8a6-8bc0-55d3-81ae-4d85820948969120.011.0640664[[CLS], [VS], 51079045420, 00006494300, 177140...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, ...[0, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65...[0, 8002, 8002, 8002, 8002, 8002, 8002, 8002, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...1
5744fe3c4-9b03-55ae-ac9f-6bc4e967cde730NaNNaN8086[[CLS], [VS], 7813, 7813, 7902, 7902, 9604, 00...[1, 2, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29...[0, 7582, 7582, 7582, 7582, 7582, 7582, 7582, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
.............................................
173666cf2115d7-937e-511d-b159-dd7eb3d5d42030NaNNaN166172[[CLS], [VS], 66591018442, 63323026201, 001350...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32...[0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
17366731338a39-28f9-54a5-a810-2d05fbaa516640NaNNaN283292[[CLS], [VS], 5014, 5123, 00338011704, 6332302...[1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50...[0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
1736680989415d-394c-5f42-8dac-75dc7306a23c30NaNNaN470476[[CLS], [VS], 51079043620, 51079088120, 492810...[1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 4, 2, 7, 7, 7, ...[0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0, 52,...[0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, ...0
17366926fb8fef-b976-5c55-859d-cc190261f94b30NaNNaN9399[[CLS], [VS], 0SRD0J9, 60505251903, 0090422446...[1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61...[0, 5085, 5085, 5085, 5085, 5085, 5085, 5085, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
17367002cee673-6875-50c9-bad9-e7a7746731eb30NaNNaN98104[[CLS], [VS], 49502069724, 00409490234, 003380...[1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...[0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52...[0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...[0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...0
\n", - "

163671 rows × 14 columns

\n", - "
" - ], - "text/plain": [ - " patient_id num_visits deceased \\\n", - "0 f8f3289a-057f-5fcc-a714-5f6109ca16c4 2 0 \n", - "1 9b62c9f4-3fdc-5020-82b5-ae5b8292445a 4 0 \n", - "2 2ca522eb-dd89-5f79-8155-9599ea46b0b2 2 1 \n", - "4 02adf8a6-8bc0-55d3-81ae-4d8582094896 9 1 \n", - "5 744fe3c4-9b03-55ae-ac9f-6bc4e967cde7 3 0 \n", - "... ... ... ... \n", - "173666 cf2115d7-937e-511d-b159-dd7eb3d5d420 3 0 \n", - "173667 31338a39-28f9-54a5-a810-2d05fbaa5166 4 0 \n", - "173668 0989415d-394c-5f42-8dac-75dc7306a23c 3 0 \n", - "173669 26fb8fef-b976-5c55-859d-cc190261f94b 3 0 \n", - "173670 02cee673-6875-50c9-bad9-e7a7746731eb 3 0 \n", - "\n", - " death_after_start death_after_end length token_length \\\n", - "0 NaN NaN 1 4 \n", - "1 NaN NaN 43 52 \n", - "2 244.0 242.0 51 54 \n", - "4 20.0 11.0 640 664 \n", - "5 NaN NaN 80 86 \n", - "... ... ... ... ... \n", - "173666 NaN NaN 166 172 \n", - "173667 NaN NaN 283 292 \n", - "173668 NaN NaN 470 476 \n", - "173669 NaN NaN 93 99 \n", - "173670 NaN NaN 98 104 \n", - "\n", - " event_tokens_2048 \\\n", - "0 [[CLS], [VS], 8938, [VE], [PAD], [PAD], [PAD],... \n", - "1 [[CLS], [VS], 7569, 66689036430, 00904224461, ... \n", - "2 [[CLS], [VS], 00904629261, 00904642281, 009046... \n", - "4 [[CLS], [VS], 51079045420, 00006494300, 177140... \n", - "5 [[CLS], [VS], 7813, 7813, 7902, 7902, 9604, 00... \n", - "... ... \n", - "173666 [[CLS], [VS], 66591018442, 63323026201, 001350... \n", - "173667 [[CLS], [VS], 5014, 5123, 00338011704, 6332302... \n", - "173668 [[CLS], [VS], 51079043620, 51079088120, 492810... \n", - "173669 [[CLS], [VS], 0SRD0J9, 60505251903, 0090422446... \n", - "173670 [[CLS], [VS], 49502069724, 00409490234, 003380... \n", - "\n", - " type_tokens_2048 \\\n", - "0 [1, 2, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 3, 4, ... \n", - "2 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "4 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 5, 5, ... \n", - "5 [1, 2, 7, 7, 7, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "... ... \n", - "173666 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "173667 [1, 2, 7, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "173668 [1, 2, 6, 6, 6, 6, 6, 6, 6, 3, 4, 2, 7, 7, 7, ... \n", - "173669 [1, 2, 7, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "173670 [1, 2, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, ... \n", - "\n", - " age_tokens_2048 \\\n", - "0 [0, 18, 18, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28... \n", - "2 [0, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86, 86... \n", - "4 [0, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65... \n", - "5 [0, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29... \n", - "... ... \n", - "173666 [0, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32... \n", - "173667 [0, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50... \n", - "173668 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 0, 52,... \n", - "173669 [0, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61... \n", - "173670 [0, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52, 52... \n", - "\n", - " time_tokens_2048 \\\n", - "0 [0, 8262, 8262, 8262, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 5963, 5963, 5963, 5963, 5963, 5963, 5963, ... \n", - "2 [0, 8016, 8016, 8016, 8016, 8016, 8016, 8016, ... \n", - "4 [0, 8002, 8002, 8002, 8002, 8002, 8002, 8002, ... \n", - "5 [0, 7582, 7582, 7582, 7582, 7582, 7582, 7582, ... \n", - "... ... \n", - "173666 [0, 5481, 5481, 5481, 5481, 5481, 5481, 5481, ... \n", - "173667 [0, 4997, 4997, 4997, 4997, 4997, 4997, 4997, ... \n", - "173668 [0, 5309, 5309, 5309, 5309, 5309, 5309, 5309, ... \n", - "173669 [0, 5085, 5085, 5085, 5085, 5085, 5085, 5085, ... \n", - "173670 [0, 5642, 5642, 5642, 5642, 5642, 5642, 5642, ... \n", - "\n", - " visit_tokens_2048 \\\n", - "0 [0, 2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, ... \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "5 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "... ... \n", - "173666 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "173667 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "173668 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 1, 1, 1, ... \n", - "173669 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "173670 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... \n", - "\n", - " position_tokens_2048 label \n", - "0 [0, 2, 2, 2, 2049, 2049, 2049, 2049, 2049, 204... 0 \n", - "1 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "2 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "4 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 1 \n", - "5 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "... ... ... \n", - "173666 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "173667 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "173668 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, ... 0 \n", - "173669 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "173670 [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ... 0 \n", - "\n", - "[163671 rows x 14 columns]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "metadata": {}, + "outputs": [], "source": [ "# Load data\n", "patients = pd.read_parquet(DATA_PATH)\n", @@ -452,9 +76,7 @@ "cell_type": "code", "execution_count": null, "id": "a789c95079b33513", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Create dictionary of all possible medical concepts\n", @@ -474,9 +96,7 @@ "cell_type": "code", "execution_count": null, "id": "653b48b3eeb19fd3", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Create the tokenizer dictionary\n", @@ -498,9 +118,7 @@ "cell_type": "code", "execution_count": null, "id": "4277ab7dfe592f7f", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Test Examples\n", @@ -514,9 +132,7 @@ "cell_type": "code", "execution_count": null, "id": "c1a005251c7b7b67", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Save tokenizer\n", @@ -527,9 +143,7 @@ "cell_type": "code", "execution_count": null, "id": "bb77624eb92b2132", - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [], "source": [ "# Create tokenizer\n", From 37473304a68c3ffbbb87e741e92c37a5e2fa7543 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 21:03:04 +0000 Subject: [PATCH 8/8] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2aa772d..b2d9345 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: - id: check-yaml - id: check-toml - - repo: https://github.com/charliermarsh/ruff-pre-commit + - repo: https://github.com/astral-sh/ruff-pre-commit rev: 'v0.4.2' hooks: - id: ruff