important_words/Noun Detector Train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c5001936",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (21.2.4)\n",
      "Requirement already satisfied: pyconll in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (3.1.0)\n",
      "Requirement already satisfied: transformers in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (4.9.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (2020.11.13)\n",
      "Requirement already satisfied: packaging in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: dataclasses in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (0.8)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (4.61.2)\n",
      "Requirement already satisfied: huggingface-hub==0.0.12 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (0.0.12)\n",
      "Requirement already satisfied: filelock in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (1.19.2)\n",
      "Requirement already satisfied: sacremoses in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (0.0.45)\n",
      "Requirement already satisfied: importlib-metadata in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from transformers) (3.7.0)\n",
      "Requirement already satisfied: typing-extensions in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from importlib-metadata->transformers) (3.4.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests->transformers) (1.26.6)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: click in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: six in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: joblib in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: dotmap in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (1.3.24)\n",
      "Requirement already satisfied: spacy in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (3.1.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (20.9)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (4.61.2)\n",
      "Requirement already satisfied: setuptools in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (49.6.0.post20210108)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (1.19.2)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (0.6.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (1.8.2)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.7 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (3.0.8)\n",
      "Requirement already satisfied: typer<0.4.0,>=0.3.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (0.3.2)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (0.7.4)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.8 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (8.0.8)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (0.8.2)\n",
      "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (3.7.4.3)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (2.25.1)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (2.4.1)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (2.0.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (1.0.5)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (2.0.6)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (3.0.5)\n",
      "Requirement already satisfied: jinja2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from spacy) (2.11.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from catalogue<2.1.0,>=2.0.4->spacy) (3.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from packaging>=20.0->spacy) (2.4.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pathy>=0.3.5->spacy) (5.2.0)\n",
      "Requirement already satisfied: dataclasses<1.0,>=0.6 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from pathy>=0.3.5->spacy) (0.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.6)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: contextvars<3,>=2.4 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from thinc<8.1.0,>=8.0.8->spacy) (2.4)\n",
      "Requirement already satisfied: immutables>=0.9 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from contextvars<3,>=2.4->thinc<8.1.0,>=8.0.8->spacy) (0.15)\n",
      "Requirement already satisfied: click<7.2.0,>=7.1.1 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from typer<0.4.0,>=0.3.0->spacy) (7.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/ec2-user/anaconda3/envs/pytorch_latest_p36/lib/python3.6/site-packages (from jinja2->spacy) (1.1.1)\n"
     ]
    }
   ],
   "source": [
    "! pip install pip --upgrade\n",
    "! pip install pyconll\n",
    "! pip install transformers\n",
    "! pip install dotmap\n",
    "! pip install spacy\n",
    "# ! pip install torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d6295f7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "from dotmap import DotMap\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] =\"0\"\n",
    "import torch\n",
    "from transformers import (RobertaConfig, RobertaModel, RobertaTokenizer)\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "from dataset import Dataset\n",
    "from train import train\n",
    "from model import Roberta\n",
    "from evaluate import evaluate\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a15ef8cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set arguments\n"
     ]
    }
   ],
   "source": [
    "args = {\n",
    "    \"loadmodel_name_path\": \"roberta-base\",#\"trained_models/nouns_propnouns/\",#roberta-base\n",
    "    \"savemodel_path\":\"trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc2/\",\n",
    "    \n",
    "    # use datasets\n",
    "    \"UD\":True,\n",
    "    \"tpkoc\":False,\n",
    "    \"conll2003\":True,\n",
    "    \"GMB\":True,\n",
    "    \n",
    "    \"local_rank\":-1,\n",
    "    \"n_gpu\":0,\n",
    "    \"device\":\"cuda\" if torch.cuda.is_available() else \"cpu\",\n",
    "    \"do_lower_case\":True,\n",
    "    \"per_gpu_train_batch_size\": 16,\n",
    "    \"per_gpu_eval_batch_size\": 1,\n",
    "    \"gradient_accumulation_steps\": 1,\n",
    "    \"max_candidates_per_forward_eval\": 1,\n",
    "    \"max_seq_length\":192,\n",
    "\n",
    "    \"learning_rate\": 5e-6,\n",
    "    \"adam_epsilon\": 1e-8,\n",
    "    \"max_grad_norm\": 1.0,\n",
    "\n",
    "    \"num_train_epochs\": 3,\n",
    "    \"warmup_steps\": 500,\n",
    "\n",
    "    \"fp16\": \"\",\n",
    "    \"output_dir\":\"outputs/\",\n",
    "\n",
    "    \"seed\": 50\n",
    "}\n",
    "args = DotMap(args)\n",
    "print(\"set arguments\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "807289ee",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-base were not used when initializing Roberta: ['lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.weight', 'lm_head.decoder.weight']\n",
      "- This IS expected if you are initializing Roberta from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing Roberta from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of Roberta were not initialized from the model checkpoint at roberta-base and are newly initialized: ['encoder.layer.1.output.dense.weight', 'encoder.layer.4.attention.self.query.weight', 'encoder.layer.6.attention.self.query.bias', 'encoder.layer.7.attention.output.LayerNorm.weight', 'encoder.layer.3.output.dense.bias', 'encoder.layer.9.attention.self.value.weight', 'embeddings.word_embeddings.weight', 'encoder.layer.10.attention.output.dense.weight', 'embeddings.position_embeddings.weight', 'pooler.dense.weight', 'encoder.layer.6.attention.self.value.bias', 'encoder.layer.10.attention.output.dense.bias', 'encoder.layer.11.attention.output.LayerNorm.weight', 'encoder.layer.5.attention.output.LayerNorm.weight', 'encoder.layer.6.attention.self.value.weight', 'encoder.layer.10.output.LayerNorm.weight', 'encoder.layer.6.output.LayerNorm.weight', 'encoder.layer.10.attention.self.value.weight', 'encoder.layer.4.intermediate.dense.weight', 'encoder.layer.8.output.dense.bias', 'encoder.layer.4.output.dense.weight', 'encoder.layer.7.output.dense.bias', 'embeddings.LayerNorm.weight', 'encoder.layer.6.attention.output.LayerNorm.weight', 'encoder.layer.8.attention.self.query.bias', 'encoder.layer.2.attention.self.query.bias', 'encoder.layer.11.intermediate.dense.bias', 'weights.bias', 'encoder.layer.10.intermediate.dense.weight', 'encoder.layer.11.attention.output.dense.bias', 'encoder.layer.1.attention.self.query.weight', 'encoder.layer.2.attention.output.LayerNorm.weight', 'encoder.layer.6.attention.self.key.bias', 'encoder.layer.7.attention.self.value.weight', 'encoder.layer.9.attention.self.query.bias', 'encoder.layer.0.attention.output.LayerNorm.bias', 'encoder.layer.5.attention.self.key.bias', 'encoder.layer.1.attention.self.key.weight', 'encoder.layer.9.attention.self.key.bias', 'encoder.layer.2.intermediate.dense.bias', 'encoder.layer.3.intermediate.dense.weight', 'encoder.layer.8.intermediate.dense.weight', 'encoder.layer.4.attention.output.LayerNorm.bias', 'encoder.layer.2.attention.self.query.weight', 'encoder.layer.1.output.dense.bias', 'encoder.layer.0.output.LayerNorm.bias', 'encoder.layer.8.attention.output.LayerNorm.weight', 'encoder.layer.1.attention.self.value.bias', 'encoder.layer.3.attention.self.query.weight', 'encoder.layer.2.output.LayerNorm.bias', 'encoder.layer.0.output.LayerNorm.weight', 'encoder.layer.9.attention.self.key.weight', 'encoder.layer.11.intermediate.dense.weight', 'encoder.layer.0.output.dense.bias', 'encoder.layer.1.attention.self.value.weight', 'encoder.layer.10.attention.self.value.bias', 'encoder.layer.5.output.dense.bias', 'encoder.layer.11.attention.output.LayerNorm.bias', 'encoder.layer.5.output.LayerNorm.bias', 'encoder.layer.9.intermediate.dense.weight', 'encoder.layer.1.attention.self.key.bias', 'embeddings.LayerNorm.bias', 'encoder.layer.10.attention.self.query.weight', 'encoder.layer.10.attention.output.LayerNorm.bias', 'encoder.layer.1.attention.output.dense.bias', 'encoder.layer.5.attention.output.dense.weight', 'encoder.layer.7.output.dense.weight', 'encoder.layer.7.output.LayerNorm.weight', 'encoder.layer.10.attention.self.key.weight', 'encoder.layer.3.output.dense.weight', 'encoder.layer.8.intermediate.dense.bias', 'encoder.layer.4.attention.self.query.bias', 'encoder.layer.7.attention.output.LayerNorm.bias', 'encoder.layer.9.output.dense.weight', 'encoder.layer.10.output.dense.weight', 'encoder.layer.6.attention.output.dense.weight', 'encoder.layer.8.attention.self.key.bias', 'encoder.layer.9.attention.output.dense.weight', 'encoder.layer.4.output.LayerNorm.weight', 'encoder.layer.5.attention.self.query.weight', 'encoder.layer.3.intermediate.dense.bias', 'encoder.layer.7.attention.self.key.bias', 'encoder.layer.4.attention.output.LayerNorm.weight', 'encoder.layer.10.output.dense.bias', 'encoder.layer.2.attention.self.value.bias', 'encoder.layer.11.attention.self.query.bias', 'encoder.layer.6.output.dense.bias', 'encoder.layer.8.output.LayerNorm.bias', 'encoder.layer.9.attention.output.LayerNorm.bias', 'embeddings.token_type_embeddings.weight', 'encoder.layer.11.attention.self.value.weight', 'encoder.layer.4.output.LayerNorm.bias', 'encoder.layer.9.attention.self.value.bias', 'encoder.layer.2.attention.output.dense.bias', 'encoder.layer.3.attention.self.value.bias', 'encoder.layer.11.output.dense.bias', 'encoder.layer.3.attention.self.query.bias', 'encoder.layer.2.attention.self.key.weight', 'encoder.layer.9.output.LayerNorm.weight', 'encoder.layer.5.intermediate.dense.bias', 'encoder.layer.0.attention.self.query.weight', 'encoder.layer.7.attention.self.query.weight', 'encoder.layer.3.attention.output.dense.weight', 'encoder.layer.0.attention.self.value.weight', 'encoder.layer.8.attention.output.LayerNorm.bias', 'encoder.layer.4.attention.self.key.weight', 'encoder.layer.4.attention.output.dense.weight', 'encoder.layer.11.attention.self.value.bias', 'encoder.layer.5.attention.self.key.weight', 'encoder.layer.3.attention.self.key.bias', 'encoder.layer.2.attention.output.dense.weight', 'encoder.layer.0.attention.output.dense.weight', 'encoder.layer.3.attention.output.dense.bias', 'encoder.layer.0.intermediate.dense.bias', 'encoder.layer.11.output.dense.weight', 'encoder.layer.3.attention.output.LayerNorm.weight', 'encoder.layer.10.attention.self.key.bias', 'encoder.layer.7.attention.output.dense.weight', 'encoder.layer.5.intermediate.dense.weight', 'encoder.layer.1.output.LayerNorm.bias', 'encoder.layer.4.attention.self.key.bias', 'encoder.layer.10.intermediate.dense.bias', 'encoder.layer.0.attention.self.value.bias', 'encoder.layer.7.attention.self.key.weight', 'encoder.layer.4.attention.output.dense.bias', 'encoder.layer.2.intermediate.dense.weight', 'encoder.layer.4.output.dense.bias', 'encoder.layer.2.output.dense.weight', 'encoder.layer.7.attention.self.value.bias', 'encoder.layer.9.attention.self.query.weight', 'encoder.layer.1.attention.output.dense.weight', 'encoder.layer.1.attention.output.LayerNorm.weight', 'encoder.layer.4.intermediate.dense.bias', 'encoder.layer.4.attention.self.value.bias', 'encoder.layer.9.attention.output.dense.bias', 'encoder.layer.10.output.LayerNorm.bias', 'encoder.layer.7.intermediate.dense.weight', 'encoder.layer.0.attention.output.LayerNorm.weight', 'encoder.layer.6.output.dense.weight', 'encoder.layer.0.output.dense.weight', 'pooler.dense.bias', 'encoder.layer.8.attention.self.value.weight', 'encoder.layer.1.attention.output.LayerNorm.bias', 'encoder.layer.1.intermediate.dense.weight', 'encoder.layer.8.attention.self.query.weight', 'encoder.layer.9.output.LayerNorm.bias', 'encoder.layer.1.intermediate.dense.bias', 'encoder.layer.11.attention.self.key.bias', 'encoder.layer.10.attention.output.LayerNorm.weight', 'weights.weight', 'encoder.layer.3.attention.output.LayerNorm.bias', 'encoder.layer.8.attention.output.dense.weight', 'encoder.layer.6.attention.output.dense.bias', 'encoder.layer.2.output.LayerNorm.weight', 'encoder.layer.0.attention.self.query.bias', 'encoder.layer.6.output.LayerNorm.bias', 'encoder.layer.3.output.LayerNorm.bias', 'encoder.layer.11.output.LayerNorm.weight', 'encoder.layer.5.attention.self.value.bias', 'encoder.layer.7.attention.output.dense.bias', 'encoder.layer.11.attention.self.query.weight', 'encoder.layer.1.output.LayerNorm.weight', 'encoder.layer.1.attention.self.query.bias', 'encoder.layer.6.attention.self.key.weight', 'encoder.layer.5.attention.self.value.weight', 'encoder.layer.5.attention.output.dense.bias', 'encoder.layer.6.intermediate.dense.bias', 'encoder.layer.2.attention.self.value.weight', 'encoder.layer.7.output.LayerNorm.bias', 'encoder.layer.2.attention.self.key.bias', 'encoder.layer.8.output.LayerNorm.weight', 'encoder.layer.5.output.dense.weight', 'encoder.layer.5.attention.output.LayerNorm.bias', 'encoder.layer.6.attention.self.query.weight', 'encoder.layer.10.attention.self.query.bias', 'encoder.layer.2.attention.output.LayerNorm.bias', 'encoder.layer.0.intermediate.dense.weight', 'encoder.layer.9.intermediate.dense.bias', 'encoder.layer.4.attention.self.value.weight', 'encoder.layer.9.attention.output.LayerNorm.weight', 'encoder.layer.2.output.dense.bias', 'encoder.layer.8.attention.output.dense.bias', 'encoder.layer.11.attention.output.dense.weight', 'encoder.layer.3.output.LayerNorm.weight', 'encoder.layer.11.attention.self.key.weight', 'encoder.layer.7.intermediate.dense.bias', 'encoder.layer.11.output.LayerNorm.bias', 'encoder.layer.5.output.LayerNorm.weight', 'encoder.layer.6.attention.output.LayerNorm.bias', 'encoder.layer.0.attention.self.key.bias', 'encoder.layer.6.intermediate.dense.weight', 'encoder.layer.7.attention.self.query.bias', 'encoder.layer.9.output.dense.bias', 'encoder.layer.5.attention.self.query.bias', 'encoder.layer.0.attention.output.dense.bias', 'encoder.layer.3.attention.self.value.weight', 'encoder.layer.8.attention.self.key.weight', 'encoder.layer.8.attention.self.value.bias', 'encoder.layer.0.attention.self.key.weight', 'encoder.layer.3.attention.self.key.weight', 'encoder.layer.8.output.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load model and tokenizer\n"
     ]
    }
   ],
   "source": [
    "tokenizer = RobertaTokenizer.from_pretrained(args.loadmodel_name_path, do_lower_case=True)\n",
    "model = Roberta.from_pretrained(args.loadmodel_name_path)\n",
    "model.to(args.device)\n",
    "print(\"load model and tokenizer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "62981c42",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c0bbfd6f43c438eb93dd00c12b9b86b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14041 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "529e40aad1ed4c27a6bb234fef7419f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43858d36fd16416190541051695e0bcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b9188efd84b44328710ec8093ac7af8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/1065 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a86f7ccfd8d40e79d49a9f822bc59fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14041 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da2b128605f24513a32cea3d8e10c5f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d24a222bc4024753b4b7aa03bf6f8190",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/1065 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "581e4aa4e3c7434f87f114759bad8b2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14041 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9068765243284dab8fc2d88f0a5f13ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4504329d1f394bfda8a6bf3f5465c618",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/1065 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e6bfa8f8dd14da996a50a7dc6b05314",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/14041 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b3d6dfc3287e43d19f755d70edb83487",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2999 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "('trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc/tokenizer_config.json',\n",
       " 'trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc/special_tokens_map.json',\n",
       " 'trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc/vocab.json',\n",
       " 'trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc/merges.txt',\n",
       " 'trained_models/scoreNN_PROP_NER_rand_conll_GMB_tpkoc/added_tokens.json')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.UD = False\n",
    "args.GMB = True\n",
    "args.tpkoc = False\n",
    "args.conll2003 = True\n",
    "model = train(args, tokenizer, model)\n",
    "model.save_pretrained(args.savemodel_path)\n",
    "tokenizer.save_pretrained(args.savemodel_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fdc11f85",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90c935fdae484cb992bb00ab98936bb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/156 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "args.UD = True\n",
    "args.GMB = False\n",
    "args.tpkoc = False\n",
    "args.conll2003 = False\n",
    "dev_d   = Dataset(args, tokenizer=tokenizer, split_type=\"dev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7b192dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "args.UD = False\n",
    "args.GMB = False\n",
    "args.tpkoc = False\n",
    "args.conll2003 = True\n",
    "dev_d2   = Dataset(args, tokenizer=tokenizer, split_type=\"dev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "432aeb3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b225b9baca64c48b9a0b143f5aa2c2d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3556 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "args.UD = False\n",
    "args.GMB = False\n",
    "args.tpkoc = True\n",
    "args.conll2003 = False\n",
    "dev_d3   = Dataset(args, tokenizer=tokenizer, split_type=\"dev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42c0f2ce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "88713223",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate(args, dev_d2, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a5c436c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate(args, dev_d, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bda5c437",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate(args, dev_d3, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ba907be",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e8f1419",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_latest_p36",
   "language": "python",
   "name": "conda_pytorch_latest_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}