example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model From huggingface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19b14664f43f469cac7f49010ce946e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "#os.environ[\"https_proxy\"] = 'http://127.0.0.1:7890'\n",
    "#os.environ[\"http_proxy\"] = 'http://127.0.0.1:7890'\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModel\n",
    "\n",
    "device = 'cuda'\n",
    "checkpoint = \"THUDM/chatglm-6b\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=True, revision = 'main')\n",
    "model = AutoModel.from_pretrained(checkpoint, trust_remote_code=True, revision = 'main')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Insert LoRA to model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "import loralib as lora\n",
    "from lora_utils.insert_lora import get_lora_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lora_config = {\n",
    "        'r': 8,\n",
    "        'lora_alpha':16,\n",
    "        'lora_dropout':0.1,\n",
    "        'enable_lora':[True, False, True],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:25<00:00,  1.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable_params:3670016 (0.06%), non_trainable_params:6173286400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = get_lora_model(model, lora_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dataset.GLM \n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "dataset.GLM.device = device\n",
    "#dataset.GLM.pad_to = 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [{'prompt':'你好', 'completion':'你好, 我是ChatGLM'}]\n",
    "pairs_encoded = dataset.GLM.encode_pairs(pairs, tokenizer)\n",
    "train_dataset = dataset.GLM.SimpleDataset(pairs_encoded)\n",
    "train_dataloader = DataLoader(dataset=train_dataset, collate_fn = dataset.GLM.collate_fn, shuffle=True, batch_size=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ChatGLMForConditionalGeneration(\n",
       "  (transformer): ChatGLMModel(\n",
       "    (word_embeddings): Embedding(130528, 4096)\n",
       "    (layers): ModuleList(\n",
       "      (0-27): 28 x GLMBlock(\n",
       "        (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "        (attention): SelfAttention(\n",
       "          (rotary_emb): RotaryEmbedding()\n",
       "          (query_key_value): LoraModel(\n",
       "            (model): QKV_layer(\n",
       "              (linear_q): Linear(\n",
       "                in_features=4096, out_features=4096, bias=True\n",
       "                (lora_dropout): Dropout(p=0.1, inplace=False)\n",
       "                (lora_A): Linear(in_features=4096, out_features=8, bias=False)\n",
       "                (lora_B): Linear(in_features=8, out_features=4096, bias=False)\n",
       "              )\n",
       "              (linear_k): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "              (linear_v): Linear(\n",
       "                in_features=4096, out_features=4096, bias=True\n",
       "                (lora_dropout): Dropout(p=0.1, inplace=False)\n",
       "                (lora_A): Linear(in_features=4096, out_features=8, bias=False)\n",
       "                (lora_B): Linear(in_features=8, out_features=4096, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (dense): Linear(in_features=4096, out_features=4096, bias=True)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): GLU(\n",
       "          (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n",
       "          (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=4096, out_features=130528, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.half().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = {k: v.to(device) for k, v in next(iter(train_dataloader)).items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(3.2227, device='cuda:0', dtype=torch.float16, grad_fn=<ToCopyBackward0>)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model(**batch).loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [\n",
    "    {'prompt':'周末适合哪里玩?', 'completion':'周末适合去上海'},\n",
    "    {'prompt':'周末适合哪里玩?', 'completion':'周末适合去北京'},\n",
    "]\n",
    "\n",
    "pairs_encoded = dataset.GLM.encode_pairs(pairs, tokenizer, with_eos=False)\n",
    "test_dataset = dataset.GLM.SimpleDataset(pairs_encoded)\n",
    "test_dataloader = DataLoader(dataset=test_dataset, collate_fn = dataset.GLM.collate_fn, shuffle=True, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch = {k: v.to(device) for k, v in next(iter(test_dataloader)).items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "outputs = model.generate(\n",
    "    **batch, \n",
    "    max_length=1024,\n",
    "    eos_token_id=130005,\n",
    "    do_sample=True,\n",
    "    temperature=0.55,\n",
    "    top_p = 0.75,\n",
    "    top_k = 10000,\n",
    "    repetition_penalty=1.5, \n",
    "    num_return_sequences=1,\n",
    "\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "周末适合哪里玩? 周末适合去北京旅游。北京是中国的首都,有着悠久的历史和丰富的文化遗产。在这里你可以参观历史古迹、博物馆和文化遗址等景点。此外,北京的特色美食也不容错过,例如烤鸭和炸酱面等等。\n",
      "如果对历史文化感兴趣的话,建议去故宫、天安门广场、颐和园和长城等地游览;如果喜欢自然风光,可以去八达岭高速或者京承高速公路上自驾游,欣赏美丽的景色。\n"
     ]
    }
   ],
   "source": [
    "for output in outputs:\n",
    "    print(tokenizer.sp_tokenizer.decode(output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "response, history = model.chat(tokenizer, \"如何缓解焦虑\", history=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'焦虑是一种较为常见的情绪反应，当人们面临不确定性、压力或风险时，可能会感到焦虑。以下是一些缓解焦虑的方法：\\n\\n1. 深呼吸：深呼吸可以帮助你放松身体和心灵。尝试缓慢地吸气，然后缓慢地呼气，重复几次。\\n\\n2. 渐进性肌肉松弛：这是一种放松身体的技巧，通过逐步收缩和松弛肌肉来减轻身体的紧张感。你可以在背部、腿部和手臂等部位练习渐进性肌肉松弛。\\n\\n3. 冥想：冥想是一种可以帮助你放松心灵和减轻焦虑的技巧。你可以通过集中注意力、呼吸、放松身体和关注内心的练习来冥想。\\n\\n4. 运动：运动可以帮助你释放紧张感和压力，同时也可以提高身体和心理的健康状况。你可以尝试跑步、瑜伽、游泳等运动。\\n\\n5. 寻求支持：与家人、朋友或专业人士谈论你的问题可以帮助你减轻焦虑。你可以寻求心理咨询或与支持团体联系。\\n\\n6. 改善生活方式：保持健康的饮食、充足的睡眠和规律的锻炼可以帮助你改善身体和心理的健康状况。\\n\\n请注意，如果你的焦虑症状持续很长时间或影响到你的日常生活，请寻求专业医疗帮助。'"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load pretrain weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_IncompatibleKeys(missing_keys=['transformer.word_embeddings.weight', 'transformer.layers.0.input_layernorm.weight', 'transformer.layers.0.input_layernorm.bias', 'transformer.layers.0.attention.query_key_value.model.linear_q.weight', 'transformer.layers.0.attention.query_key_value.model.linear_q.bias', 'transformer.layers.0.attention.query_key_value.model.linear_k.weight', 'transformer.layers.0.attention.query_key_value.model.linear_k.bias', 'transformer.layers.0.attention.query_key_value.model.linear_v.weight', 'transformer.layers.0.attention.query_key_value.model.linear_v.bias', 'transformer.layers.0.attention.dense.weight', 'transformer.layers.0.attention.dense.bias', 'transformer.layers.0.post_attention_layernorm.weight', 'transformer.layers.0.post_attention_layernorm.bias', 'transformer.layers.0.mlp.dense_h_to_4h.weight', 'transformer.layers.0.mlp.dense_h_to_4h.bias', 'transformer.layers.0.mlp.dense_4h_to_h.weight', 'transformer.layers.0.mlp.dense_4h_to_h.bias', 'transformer.layers.1.input_layernorm.weight', 'transformer.layers.1.input_layernorm.bias', 'transformer.layers.1.attention.query_key_value.model.linear_q.weight', 'transformer.layers.1.attention.query_key_value.model.linear_q.bias', 'transformer.layers.1.attention.query_key_value.model.linear_k.weight', 'transformer.layers.1.attention.query_key_value.model.linear_k.bias', 'transformer.layers.1.attention.query_key_value.model.linear_v.weight', 'transformer.layers.1.attention.query_key_value.model.linear_v.bias', 'transformer.layers.1.attention.dense.weight', 'transformer.layers.1.attention.dense.bias', 'transformer.layers.1.post_attention_layernorm.weight', 'transformer.layers.1.post_attention_layernorm.bias', 'transformer.layers.1.mlp.dense_h_to_4h.weight', 'transformer.layers.1.mlp.dense_h_to_4h.bias', 'transformer.layers.1.mlp.dense_4h_to_h.weight', 'transformer.layers.1.mlp.dense_4h_to_h.bias', 'transformer.layers.2.input_layernorm.weight', 'transformer.layers.2.input_layernorm.bias', 'transformer.layers.2.attention.query_key_value.model.linear_q.weight', 'transformer.layers.2.attention.query_key_value.model.linear_q.bias', 'transformer.layers.2.attention.query_key_value.model.linear_k.weight', 'transformer.layers.2.attention.query_key_value.model.linear_k.bias', 'transformer.layers.2.attention.query_key_value.model.linear_v.weight', 'transformer.layers.2.attention.query_key_value.model.linear_v.bias', 'transformer.layers.2.attention.dense.weight', 'transformer.layers.2.attention.dense.bias', 'transformer.layers.2.post_attention_layernorm.weight', 'transformer.layers.2.post_attention_layernorm.bias', 'transformer.layers.2.mlp.dense_h_to_4h.weight', 'transformer.layers.2.mlp.dense_h_to_4h.bias', 'transformer.layers.2.mlp.dense_4h_to_h.weight', 'transformer.layers.2.mlp.dense_4h_to_h.bias', 'transformer.layers.3.input_layernorm.weight', 'transformer.layers.3.input_layernorm.bias', 'transformer.layers.3.attention.query_key_value.model.linear_q.weight', 'transformer.layers.3.attention.query_key_value.model.linear_q.bias', 'transformer.layers.3.attention.query_key_value.model.linear_k.weight', 'transformer.layers.3.attention.query_key_value.model.linear_k.bias', 'transformer.layers.3.attention.query_key_value.model.linear_v.weight', 'transformer.layers.3.attention.query_key_value.model.linear_v.bias', 'transformer.layers.3.attention.dense.weight', 'transformer.layers.3.attention.dense.bias', 'transformer.layers.3.post_attention_layernorm.weight', 'transformer.layers.3.post_attention_layernorm.bias', 'transformer.layers.3.mlp.dense_h_to_4h.weight', 'transformer.layers.3.mlp.dense_h_to_4h.bias', 'transformer.layers.3.mlp.dense_4h_to_h.weight', 'transformer.layers.3.mlp.dense_4h_to_h.bias', 'transformer.layers.4.input_layernorm.weight', 'transformer.layers.4.input_layernorm.bias', 'transformer.layers.4.attention.query_key_value.model.linear_q.weight', 'transformer.layers.4.attention.query_key_value.model.linear_q.bias', 'transformer.layers.4.attention.query_key_value.model.linear_k.weight', 'transformer.layers.4.attention.query_key_value.model.linear_k.bias', 'transformer.layers.4.attention.query_key_value.model.linear_v.weight', 'transformer.layers.4.attention.query_key_value.model.linear_v.bias', 'transformer.layers.4.attention.dense.weight', 'transformer.layers.4.attention.dense.bias', 'transformer.layers.4.post_attention_layernorm.weight', 'transformer.layers.4.post_attention_layernorm.bias', 'transformer.layers.4.mlp.dense_h_to_4h.weight', 'transformer.layers.4.mlp.dense_h_to_4h.bias', 'transformer.layers.4.mlp.dense_4h_to_h.weight', 'transformer.layers.4.mlp.dense_4h_to_h.bias', 'transformer.layers.5.input_layernorm.weight', 'transformer.layers.5.input_layernorm.bias', 'transformer.layers.5.attention.query_key_value.model.linear_q.weight', 'transformer.layers.5.attention.query_key_value.model.linear_q.bias', 'transformer.layers.5.attention.query_key_value.model.linear_k.weight', 'transformer.layers.5.attention.query_key_value.model.linear_k.bias', 'transformer.layers.5.attention.query_key_value.model.linear_v.weight', 'transformer.layers.5.attention.query_key_value.model.linear_v.bias', 'transformer.layers.5.attention.dense.weight', 'transformer.layers.5.attention.dense.bias', 'transformer.layers.5.post_attention_layernorm.weight', 'transformer.layers.5.post_attention_layernorm.bias', 'transformer.layers.5.mlp.dense_h_to_4h.weight', 'transformer.layers.5.mlp.dense_h_to_4h.bias', 'transformer.layers.5.mlp.dense_4h_to_h.weight', 'transformer.layers.5.mlp.dense_4h_to_h.bias', 'transformer.layers.6.input_layernorm.weight', 'transformer.layers.6.input_layernorm.bias', 'transformer.layers.6.attention.query_key_value.model.linear_q.weight', 'transformer.layers.6.attention.query_key_value.model.linear_q.bias', 'transformer.layers.6.attention.query_key_value.model.linear_k.weight', 'transformer.layers.6.attention.query_key_value.model.linear_k.bias', 'transformer.layers.6.attention.query_key_value.model.linear_v.weight', 'transformer.layers.6.attention.query_key_value.model.linear_v.bias', 'transformer.layers.6.attention.dense.weight', 'transformer.layers.6.attention.dense.bias', 'transformer.layers.6.post_attention_layernorm.weight', 'transformer.layers.6.post_attention_layernorm.bias', 'transformer.layers.6.mlp.dense_h_to_4h.weight', 'transformer.layers.6.mlp.dense_h_to_4h.bias', 'transformer.layers.6.mlp.dense_4h_to_h.weight', 'transformer.layers.6.mlp.dense_4h_to_h.bias', 'transformer.layers.7.input_layernorm.weight', 'transformer.layers.7.input_layernorm.bias', 'transformer.layers.7.attention.query_key_value.model.linear_q.weight', 'transformer.layers.7.attention.query_key_value.model.linear_q.bias', 'transformer.layers.7.attention.query_key_value.model.linear_k.weight', 'transformer.layers.7.attention.query_key_value.model.linear_k.bias', 'transformer.layers.7.attention.query_key_value.model.linear_v.weight', 'transformer.layers.7.attention.query_key_value.model.linear_v.bias', 'transformer.layers.7.attention.dense.weight', 'transformer.layers.7.attention.dense.bias', 'transformer.layers.7.post_attention_layernorm.weight', 'transformer.layers.7.post_attention_layernorm.bias', 'transformer.layers.7.mlp.dense_h_to_4h.weight', 'transformer.layers.7.mlp.dense_h_to_4h.bias', 'transformer.layers.7.mlp.dense_4h_to_h.weight', 'transformer.layers.7.mlp.dense_4h_to_h.bias', 'transformer.layers.8.input_layernorm.weight', 'transformer.layers.8.input_layernorm.bias', 'transformer.layers.8.attention.query_key_value.model.linear_q.weight', 'transformer.layers.8.attention.query_key_value.model.linear_q.bias', 'transformer.layers.8.attention.query_key_value.model.linear_k.weight', 'transformer.layers.8.attention.query_key_value.model.linear_k.bias', 'transformer.layers.8.attention.query_key_value.model.linear_v.weight', 'transformer.layers.8.attention.query_key_value.model.linear_v.bias', 'transformer.layers.8.attention.dense.weight', 'transformer.layers.8.attention.dense.bias', 'transformer.layers.8.post_attention_layernorm.weight', 'transformer.layers.8.post_attention_layernorm.bias', 'transformer.layers.8.mlp.dense_h_to_4h.weight', 'transformer.layers.8.mlp.dense_h_to_4h.bias', 'transformer.layers.8.mlp.dense_4h_to_h.weight', 'transformer.layers.8.mlp.dense_4h_to_h.bias', 'transformer.layers.9.input_layernorm.weight', 'transformer.layers.9.input_layernorm.bias', 'transformer.layers.9.attention.query_key_value.model.linear_q.weight', 'transformer.layers.9.attention.query_key_value.model.linear_q.bias', 'transformer.layers.9.attention.query_key_value.model.linear_k.weight', 'transformer.layers.9.attention.query_key_value.model.linear_k.bias', 'transformer.layers.9.attention.query_key_value.model.linear_v.weight', 'transformer.layers.9.attention.query_key_value.model.linear_v.bias', 'transformer.layers.9.attention.dense.weight', 'transformer.layers.9.attention.dense.bias', 'transformer.layers.9.post_attention_layernorm.weight', 'transformer.layers.9.post_attention_layernorm.bias', 'transformer.layers.9.mlp.dense_h_to_4h.weight', 'transformer.layers.9.mlp.dense_h_to_4h.bias', 'transformer.layers.9.mlp.dense_4h_to_h.weight', 'transformer.layers.9.mlp.dense_4h_to_h.bias', 'transformer.layers.10.input_layernorm.weight', 'transformer.layers.10.input_layernorm.bias', 'transformer.layers.10.attention.query_key_value.model.linear_q.weight', 'transformer.layers.10.attention.query_key_value.model.linear_q.bias', 'transformer.layers.10.attention.query_key_value.model.linear_k.weight', 'transformer.layers.10.attention.query_key_value.model.linear_k.bias', 'transformer.layers.10.attention.query_key_value.model.linear_v.weight', 'transformer.layers.10.attention.query_key_value.model.linear_v.bias', 'transformer.layers.10.attention.dense.weight', 'transformer.layers.10.attention.dense.bias', 'transformer.layers.10.post_attention_layernorm.weight', 'transformer.layers.10.post_attention_layernorm.bias', 'transformer.layers.10.mlp.dense_h_to_4h.weight', 'transformer.layers.10.mlp.dense_h_to_4h.bias', 'transformer.layers.10.mlp.dense_4h_to_h.weight', 'transformer.layers.10.mlp.dense_4h_to_h.bias', 'transformer.layers.11.input_layernorm.weight', 'transformer.layers.11.input_layernorm.bias', 'transformer.layers.11.attention.query_key_value.model.linear_q.weight', 'transformer.layers.11.attention.query_key_value.model.linear_q.bias', 'transformer.layers.11.attention.query_key_value.model.linear_k.weight', 'transformer.layers.11.attention.query_key_value.model.linear_k.bias', 'transformer.layers.11.attention.query_key_value.model.linear_v.weight', 'transformer.layers.11.attention.query_key_value.model.linear_v.bias', 'transformer.layers.11.attention.dense.weight', 'transformer.layers.11.attention.dense.bias', 'transformer.layers.11.post_attention_layernorm.weight', 'transformer.layers.11.post_attention_layernorm.bias', 'transformer.layers.11.mlp.dense_h_to_4h.weight', 'transformer.layers.11.mlp.dense_h_to_4h.bias', 'transformer.layers.11.mlp.dense_4h_to_h.weight', 'transformer.layers.11.mlp.dense_4h_to_h.bias', 'transformer.layers.12.input_layernorm.weight', 'transformer.layers.12.input_layernorm.bias', 'transformer.layers.12.attention.query_key_value.model.linear_q.weight', 'transformer.layers.12.attention.query_key_value.model.linear_q.bias', 'transformer.layers.12.attention.query_key_value.model.linear_k.weight', 'transformer.layers.12.attention.query_key_value.model.linear_k.bias', 'transformer.layers.12.attention.query_key_value.model.linear_v.weight', 'transformer.layers.12.attention.query_key_value.model.linear_v.bias', 'transformer.layers.12.attention.dense.weight', 'transformer.layers.12.attention.dense.bias', 'transformer.layers.12.post_attention_layernorm.weight', 'transformer.layers.12.post_attention_layernorm.bias', 'transformer.layers.12.mlp.dense_h_to_4h.weight', 'transformer.layers.12.mlp.dense_h_to_4h.bias', 'transformer.layers.12.mlp.dense_4h_to_h.weight', 'transformer.layers.12.mlp.dense_4h_to_h.bias', 'transformer.layers.13.input_layernorm.weight', 'transformer.layers.13.input_layernorm.bias', 'transformer.layers.13.attention.query_key_value.model.linear_q.weight', 'transformer.layers.13.attention.query_key_value.model.linear_q.bias', 'transformer.layers.13.attention.query_key_value.model.linear_k.weight', 'transformer.layers.13.attention.query_key_value.model.linear_k.bias', 'transformer.layers.13.attention.query_key_value.model.linear_v.weight', 'transformer.layers.13.attention.query_key_value.model.linear_v.bias', 'transformer.layers.13.attention.dense.weight', 'transformer.layers.13.attention.dense.bias', 'transformer.layers.13.post_attention_layernorm.weight', 'transformer.layers.13.post_attention_layernorm.bias', 'transformer.layers.13.mlp.dense_h_to_4h.weight', 'transformer.layers.13.mlp.dense_h_to_4h.bias', 'transformer.layers.13.mlp.dense_4h_to_h.weight', 'transformer.layers.13.mlp.dense_4h_to_h.bias', 'transformer.layers.14.input_layernorm.weight', 'transformer.layers.14.input_layernorm.bias', 'transformer.layers.14.attention.query_key_value.model.linear_q.weight', 'transformer.layers.14.attention.query_key_value.model.linear_q.bias', 'transformer.layers.14.attention.query_key_value.model.linear_k.weight', 'transformer.layers.14.attention.query_key_value.model.linear_k.bias', 'transformer.layers.14.attention.query_key_value.model.linear_v.weight', 'transformer.layers.14.attention.query_key_value.model.linear_v.bias', 'transformer.layers.14.attention.dense.weight', 'transformer.layers.14.attention.dense.bias', 'transformer.layers.14.post_attention_layernorm.weight', 'transformer.layers.14.post_attention_layernorm.bias', 'transformer.layers.14.mlp.dense_h_to_4h.weight', 'transformer.layers.14.mlp.dense_h_to_4h.bias', 'transformer.layers.14.mlp.dense_4h_to_h.weight', 'transformer.layers.14.mlp.dense_4h_to_h.bias', 'transformer.layers.15.input_layernorm.weight', 'transformer.layers.15.input_layernorm.bias', 'transformer.layers.15.attention.query_key_value.model.linear_q.weight', 'transformer.layers.15.attention.query_key_value.model.linear_q.bias', 'transformer.layers.15.attention.query_key_value.model.linear_k.weight', 'transformer.layers.15.attention.query_key_value.model.linear_k.bias', 'transformer.layers.15.attention.query_key_value.model.linear_v.weight', 'transformer.layers.15.attention.query_key_value.model.linear_v.bias', 'transformer.layers.15.attention.dense.weight', 'transformer.layers.15.attention.dense.bias', 'transformer.layers.15.post_attention_layernorm.weight', 'transformer.layers.15.post_attention_layernorm.bias', 'transformer.layers.15.mlp.dense_h_to_4h.weight', 'transformer.layers.15.mlp.dense_h_to_4h.bias', 'transformer.layers.15.mlp.dense_4h_to_h.weight', 'transformer.layers.15.mlp.dense_4h_to_h.bias', 'transformer.layers.16.input_layernorm.weight', 'transformer.layers.16.input_layernorm.bias', 'transformer.layers.16.attention.query_key_value.model.linear_q.weight', 'transformer.layers.16.attention.query_key_value.model.linear_q.bias', 'transformer.layers.16.attention.query_key_value.model.linear_k.weight', 'transformer.layers.16.attention.query_key_value.model.linear_k.bias', 'transformer.layers.16.attention.query_key_value.model.linear_v.weight', 'transformer.layers.16.attention.query_key_value.model.linear_v.bias', 'transformer.layers.16.attention.dense.weight', 'transformer.layers.16.attention.dense.bias', 'transformer.layers.16.post_attention_layernorm.weight', 'transformer.layers.16.post_attention_layernorm.bias', 'transformer.layers.16.mlp.dense_h_to_4h.weight', 'transformer.layers.16.mlp.dense_h_to_4h.bias', 'transformer.layers.16.mlp.dense_4h_to_h.weight', 'transformer.layers.16.mlp.dense_4h_to_h.bias', 'transformer.layers.17.input_layernorm.weight', 'transformer.layers.17.input_layernorm.bias', 'transformer.layers.17.attention.query_key_value.model.linear_q.weight', 'transformer.layers.17.attention.query_key_value.model.linear_q.bias', 'transformer.layers.17.attention.query_key_value.model.linear_k.weight', 'transformer.layers.17.attention.query_key_value.model.linear_k.bias', 'transformer.layers.17.attention.query_key_value.model.linear_v.weight', 'transformer.layers.17.attention.query_key_value.model.linear_v.bias', 'transformer.layers.17.attention.dense.weight', 'transformer.layers.17.attention.dense.bias', 'transformer.layers.17.post_attention_layernorm.weight', 'transformer.layers.17.post_attention_layernorm.bias', 'transformer.layers.17.mlp.dense_h_to_4h.weight', 'transformer.layers.17.mlp.dense_h_to_4h.bias', 'transformer.layers.17.mlp.dense_4h_to_h.weight', 'transformer.layers.17.mlp.dense_4h_to_h.bias', 'transformer.layers.18.input_layernorm.weight', 'transformer.layers.18.input_layernorm.bias', 'transformer.layers.18.attention.query_key_value.model.linear_q.weight', 'transformer.layers.18.attention.query_key_value.model.linear_q.bias', 'transformer.layers.18.attention.query_key_value.model.linear_k.weight', 'transformer.layers.18.attention.query_key_value.model.linear_k.bias', 'transformer.layers.18.attention.query_key_value.model.linear_v.weight', 'transformer.layers.18.attention.query_key_value.model.linear_v.bias', 'transformer.layers.18.attention.dense.weight', 'transformer.layers.18.attention.dense.bias', 'transformer.layers.18.post_attention_layernorm.weight', 'transformer.layers.18.post_attention_layernorm.bias', 'transformer.layers.18.mlp.dense_h_to_4h.weight', 'transformer.layers.18.mlp.dense_h_to_4h.bias', 'transformer.layers.18.mlp.dense_4h_to_h.weight', 'transformer.layers.18.mlp.dense_4h_to_h.bias', 'transformer.layers.19.input_layernorm.weight', 'transformer.layers.19.input_layernorm.bias', 'transformer.layers.19.attention.query_key_value.model.linear_q.weight', 'transformer.layers.19.attention.query_key_value.model.linear_q.bias', 'transformer.layers.19.attention.query_key_value.model.linear_k.weight', 'transformer.layers.19.attention.query_key_value.model.linear_k.bias', 'transformer.layers.19.attention.query_key_value.model.linear_v.weight', 'transformer.layers.19.attention.query_key_value.model.linear_v.bias', 'transformer.layers.19.attention.dense.weight', 'transformer.layers.19.attention.dense.bias', 'transformer.layers.19.post_attention_layernorm.weight', 'transformer.layers.19.post_attention_layernorm.bias', 'transformer.layers.19.mlp.dense_h_to_4h.weight', 'transformer.layers.19.mlp.dense_h_to_4h.bias', 'transformer.layers.19.mlp.dense_4h_to_h.weight', 'transformer.layers.19.mlp.dense_4h_to_h.bias', 'transformer.layers.20.input_layernorm.weight', 'transformer.layers.20.input_layernorm.bias', 'transformer.layers.20.attention.query_key_value.model.linear_q.weight', 'transformer.layers.20.attention.query_key_value.model.linear_q.bias', 'transformer.layers.20.attention.query_key_value.model.linear_k.weight', 'transformer.layers.20.attention.query_key_value.model.linear_k.bias', 'transformer.layers.20.attention.query_key_value.model.linear_v.weight', 'transformer.layers.20.attention.query_key_value.model.linear_v.bias', 'transformer.layers.20.attention.dense.weight', 'transformer.layers.20.attention.dense.bias', 'transformer.layers.20.post_attention_layernorm.weight', 'transformer.layers.20.post_attention_layernorm.bias', 'transformer.layers.20.mlp.dense_h_to_4h.weight', 'transformer.layers.20.mlp.dense_h_to_4h.bias', 'transformer.layers.20.mlp.dense_4h_to_h.weight', 'transformer.layers.20.mlp.dense_4h_to_h.bias', 'transformer.layers.21.input_layernorm.weight', 'transformer.layers.21.input_layernorm.bias', 'transformer.layers.21.attention.query_key_value.model.linear_q.weight', 'transformer.layers.21.attention.query_key_value.model.linear_q.bias', 'transformer.layers.21.attention.query_key_value.model.linear_k.weight', 'transformer.layers.21.attention.query_key_value.model.linear_k.bias', 'transformer.layers.21.attention.query_key_value.model.linear_v.weight', 'transformer.layers.21.attention.query_key_value.model.linear_v.bias', 'transformer.layers.21.attention.dense.weight', 'transformer.layers.21.attention.dense.bias', 'transformer.layers.21.post_attention_layernorm.weight', 'transformer.layers.21.post_attention_layernorm.bias', 'transformer.layers.21.mlp.dense_h_to_4h.weight', 'transformer.layers.21.mlp.dense_h_to_4h.bias', 'transformer.layers.21.mlp.dense_4h_to_h.weight', 'transformer.layers.21.mlp.dense_4h_to_h.bias', 'transformer.layers.22.input_layernorm.weight', 'transformer.layers.22.input_layernorm.bias', 'transformer.layers.22.attention.query_key_value.model.linear_q.weight', 'transformer.layers.22.attention.query_key_value.model.linear_q.bias', 'transformer.layers.22.attention.query_key_value.model.linear_k.weight', 'transformer.layers.22.attention.query_key_value.model.linear_k.bias', 'transformer.layers.22.attention.query_key_value.model.linear_v.weight', 'transformer.layers.22.attention.query_key_value.model.linear_v.bias', 'transformer.layers.22.attention.dense.weight', 'transformer.layers.22.attention.dense.bias', 'transformer.layers.22.post_attention_layernorm.weight', 'transformer.layers.22.post_attention_layernorm.bias', 'transformer.layers.22.mlp.dense_h_to_4h.weight', 'transformer.layers.22.mlp.dense_h_to_4h.bias', 'transformer.layers.22.mlp.dense_4h_to_h.weight', 'transformer.layers.22.mlp.dense_4h_to_h.bias', 'transformer.layers.23.input_layernorm.weight', 'transformer.layers.23.input_layernorm.bias', 'transformer.layers.23.attention.query_key_value.model.linear_q.weight', 'transformer.layers.23.attention.query_key_value.model.linear_q.bias', 'transformer.layers.23.attention.query_key_value.model.linear_k.weight', 'transformer.layers.23.attention.query_key_value.model.linear_k.bias', 'transformer.layers.23.attention.query_key_value.model.linear_v.weight', 'transformer.layers.23.attention.query_key_value.model.linear_v.bias', 'transformer.layers.23.attention.dense.weight', 'transformer.layers.23.attention.dense.bias', 'transformer.layers.23.post_attention_layernorm.weight', 'transformer.layers.23.post_attention_layernorm.bias', 'transformer.layers.23.mlp.dense_h_to_4h.weight', 'transformer.layers.23.mlp.dense_h_to_4h.bias', 'transformer.layers.23.mlp.dense_4h_to_h.weight', 'transformer.layers.23.mlp.dense_4h_to_h.bias', 'transformer.layers.24.input_layernorm.weight', 'transformer.layers.24.input_layernorm.bias', 'transformer.layers.24.attention.query_key_value.model.linear_q.weight', 'transformer.layers.24.attention.query_key_value.model.linear_q.bias', 'transformer.layers.24.attention.query_key_value.model.linear_k.weight', 'transformer.layers.24.attention.query_key_value.model.linear_k.bias', 'transformer.layers.24.attention.query_key_value.model.linear_v.weight', 'transformer.layers.24.attention.query_key_value.model.linear_v.bias', 'transformer.layers.24.attention.dense.weight', 'transformer.layers.24.attention.dense.bias', 'transformer.layers.24.post_attention_layernorm.weight', 'transformer.layers.24.post_attention_layernorm.bias', 'transformer.layers.24.mlp.dense_h_to_4h.weight', 'transformer.layers.24.mlp.dense_h_to_4h.bias', 'transformer.layers.24.mlp.dense_4h_to_h.weight', 'transformer.layers.24.mlp.dense_4h_to_h.bias', 'transformer.layers.25.input_layernorm.weight', 'transformer.layers.25.input_layernorm.bias', 'transformer.layers.25.attention.query_key_value.model.linear_q.weight', 'transformer.layers.25.attention.query_key_value.model.linear_q.bias', 'transformer.layers.25.attention.query_key_value.model.linear_k.weight', 'transformer.layers.25.attention.query_key_value.model.linear_k.bias', 'transformer.layers.25.attention.query_key_value.model.linear_v.weight', 'transformer.layers.25.attention.query_key_value.model.linear_v.bias', 'transformer.layers.25.attention.dense.weight', 'transformer.layers.25.attention.dense.bias', 'transformer.layers.25.post_attention_layernorm.weight', 'transformer.layers.25.post_attention_layernorm.bias', 'transformer.layers.25.mlp.dense_h_to_4h.weight', 'transformer.layers.25.mlp.dense_h_to_4h.bias', 'transformer.layers.25.mlp.dense_4h_to_h.weight', 'transformer.layers.25.mlp.dense_4h_to_h.bias', 'transformer.layers.26.input_layernorm.weight', 'transformer.layers.26.input_layernorm.bias', 'transformer.layers.26.attention.query_key_value.model.linear_q.weight', 'transformer.layers.26.attention.query_key_value.model.linear_q.bias', 'transformer.layers.26.attention.query_key_value.model.linear_k.weight', 'transformer.layers.26.attention.query_key_value.model.linear_k.bias', 'transformer.layers.26.attention.query_key_value.model.linear_v.weight', 'transformer.layers.26.attention.query_key_value.model.linear_v.bias', 'transformer.layers.26.attention.dense.weight', 'transformer.layers.26.attention.dense.bias', 'transformer.layers.26.post_attention_layernorm.weight', 'transformer.layers.26.post_attention_layernorm.bias', 'transformer.layers.26.mlp.dense_h_to_4h.weight', 'transformer.layers.26.mlp.dense_h_to_4h.bias', 'transformer.layers.26.mlp.dense_4h_to_h.weight', 'transformer.layers.26.mlp.dense_4h_to_h.bias', 'transformer.layers.27.input_layernorm.weight', 'transformer.layers.27.input_layernorm.bias', 'transformer.layers.27.attention.query_key_value.model.linear_q.weight', 'transformer.layers.27.attention.query_key_value.model.linear_q.bias', 'transformer.layers.27.attention.query_key_value.model.linear_k.weight', 'transformer.layers.27.attention.query_key_value.model.linear_k.bias', 'transformer.layers.27.attention.query_key_value.model.linear_v.weight', 'transformer.layers.27.attention.query_key_value.model.linear_v.bias', 'transformer.layers.27.attention.dense.weight', 'transformer.layers.27.attention.dense.bias', 'transformer.layers.27.post_attention_layernorm.weight', 'transformer.layers.27.post_attention_layernorm.bias', 'transformer.layers.27.mlp.dense_h_to_4h.weight', 'transformer.layers.27.mlp.dense_h_to_4h.bias', 'transformer.layers.27.mlp.dense_4h_to_h.weight', 'transformer.layers.27.mlp.dense_4h_to_h.bias', 'transformer.final_layernorm.weight', 'transformer.final_layernorm.bias', 'lm_head.weight'], unexpected_keys=[])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('saved/chatglm-6b_alpaca_5.pt'), strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.16 ('gpt': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6e50d88e1ef9972f89e57743e3bfd7b166432ae8f03786cd5ae9ea181ff74793"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}