Skip to content
This repository has been archived by the owner on Jan 15, 2024. It is now read-only.

Commit

Permalink
Fix language model ipynb (#83)
Browse files Browse the repository at this point in the history 
* Fix language model ipynb

* Doc fix in word_embedding.ipynb

* Fix wemb eval paths

* Fix warning

* Rerun language_model.ipynb
  • Loading branch information
@leezu @szha
leezu authored and szha committed Apr 25, 2018
1 parent 94088d4 commit f9edbfe
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 63 deletions.
121 changes: 60 additions & 61 deletions docs/examples/language_model/language_model.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import time\n",
"import math\n",
"\n",
Expand Down Expand Up @@ -128,15 +131,14 @@
"output_type": "stream",
"text": [
"StandardRNN(\n",
" (decoder): HybridSequential(\n",
" (0): Dropout(p = 0.2, axes=())\n",
" (1): Dense(200 -> 33278, linear)\n",
" )\n",
" (embedding): HybridSequential(\n",
" (0): Embedding(33278 -> 200, float32)\n",
" (1): Dropout(p = 0.2, axes=())\n",
" )\n",
" (encoder): LSTM(200 -> 800, TNC, num_layers=2, dropout=0.2)\n",
" (encoder): LSTM(200 -> 200.0, TNC, num_layers=2, dropout=0.2)\n",
" (decoder): HybridSequential(\n",
" (0): Dense(200 -> 33278, linear)\n",
" )\n",
")\n",
"Vocab(size=33278, unk=\"<unk>\", reserved=\"['<eos>']\")\n"
]
Expand Down Expand Up @@ -239,7 +241,7 @@
"metadata": {},
"outputs": [],
"source": [
"def train(model, train_data, val_data, test_data, epochs):\n",
"def train(model, train_data, val_data, test_data, epochs, lr):\n",
" best_val = float(\"Inf\")\n",
" start_train_time = time.time()\n",
" parameters = model.collect_params().values()\n",
Expand Down Expand Up @@ -321,42 +323,42 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[Epoch 0 Batch 100/746] loss 8.05, ppl 3129.00, throughput 1395.06 samples/s\n",
"[Epoch 0 Batch 200/746] loss 7.24, ppl 1399.34, throughput 1895.63 samples/s\n",
"[Epoch 0 Batch 300/746] loss 6.96, ppl 1053.46, throughput 1996.78 samples/s\n",
"[Epoch 0 Batch 400/746] loss 6.70, ppl 813.17, throughput 1890.44 samples/s\n",
"[Epoch 0 Batch 500/746] loss 6.50, ppl 667.19, throughput 1883.28 samples/s\n",
"[Epoch 0 Batch 600/746] loss 6.33, ppl 563.24, throughput 1975.94 samples/s\n",
"[Epoch 0 Batch 700/746] loss 6.22, ppl 501.24, throughput 1930.41 samples/s\n",
"[Epoch 0] throughput 1820.23 samples/s\n",
"[Epoch 0] time cost 36.12s, valid loss 5.90, valid ppl 366.51\n",
"test loss 5.82, test ppl 337.05\n",
"[Epoch 1 Batch 100/746] loss 6.10, ppl 446.92, throughput 1932.23 samples/s\n",
"[Epoch 1 Batch 200/746] loss 6.00, ppl 404.51, throughput 1877.29 samples/s\n",
"[Epoch 1 Batch 300/746] loss 5.94, ppl 380.40, throughput 1902.84 samples/s\n",
"[Epoch 1 Batch 400/746] loss 5.89, ppl 362.86, throughput 1944.96 samples/s\n",
"[Epoch 1 Batch 500/746] loss 5.79, ppl 325.73, throughput 1867.69 samples/s\n",
"[Epoch 1 Batch 600/746] loss 5.70, ppl 298.65, throughput 1983.97 samples/s\n",
"[Epoch 1 Batch 700/746] loss 5.64, ppl 282.85, throughput 1892.81 samples/s\n",
"[Epoch 1] throughput 1915.09 samples/s\n",
"[Epoch 1] time cost 34.65s, valid loss 5.53, valid ppl 251.99\n",
"test loss 5.44, test ppl 229.78\n",
"[Epoch 2 Batch 100/746] loss 5.63, ppl 278.83, throughput 1880.85 samples/s\n",
"[Epoch 2 Batch 200/746] loss 5.56, ppl 259.11, throughput 1933.27 samples/s\n",
"[Epoch 2 Batch 300/746] loss 5.52, ppl 249.74, throughput 1858.71 samples/s\n",
"[Epoch 2 Batch 400/746] loss 5.52, ppl 249.21, throughput 1896.20 samples/s\n",
"[Epoch 2 Batch 500/746] loss 5.42, ppl 226.61, throughput 1965.41 samples/s\n",
"[Epoch 2 Batch 600/746] loss 5.36, ppl 212.60, throughput 1874.85 samples/s\n",
"[Epoch 2 Batch 700/746] loss 5.33, ppl 206.46, throughput 1890.60 samples/s\n",
"[Epoch 2] throughput 1907.40 samples/s\n",
"[Epoch 2] time cost 34.70s, valid loss 5.26, valid ppl 192.69\n",
"test loss 5.18, test ppl 177.32\n",
"Total training throughput 1534.74 samples/s\n"
"[Epoch 0 Batch 100/746] loss 8.05, ppl 3147.06, throughput 1125.76 samples/s\n",
"[Epoch 0 Batch 200/746] loss 7.25, ppl 1402.82, throughput 1201.12 samples/s\n",
"[Epoch 0 Batch 300/746] loss 6.95, ppl 1047.72, throughput 1207.49 samples/s\n",
"[Epoch 0 Batch 400/746] loss 6.70, ppl 810.72, throughput 1213.90 samples/s\n",
"[Epoch 0 Batch 500/746] loss 6.51, ppl 670.70, throughput 1214.49 samples/s\n",
"[Epoch 0 Batch 600/746] loss 6.33, ppl 562.60, throughput 1211.53 samples/s\n",
"[Epoch 0 Batch 700/746] loss 6.21, ppl 498.81, throughput 1203.75 samples/s\n",
"[Epoch 0] throughput 1195.34 samples/s\n",
"[Epoch 0] time cost 54.93s, valid loss 5.90, valid ppl 364.40\n",
"test loss 5.82, test ppl 336.80\n",
"[Epoch 1 Batch 100/746] loss 6.10, ppl 446.09, throughput 1190.65 samples/s\n",
"[Epoch 1 Batch 200/746] loss 6.00, ppl 404.99, throughput 1207.26 samples/s\n",
"[Epoch 1 Batch 300/746] loss 5.94, ppl 378.32, throughput 1200.87 samples/s\n",
"[Epoch 1 Batch 400/746] loss 5.88, ppl 356.80, throughput 1197.75 samples/s\n",
"[Epoch 1 Batch 500/746] loss 5.77, ppl 320.43, throughput 1208.13 samples/s\n",
"[Epoch 1 Batch 600/746] loss 5.68, ppl 292.73, throughput 1203.65 samples/s\n",
"[Epoch 1 Batch 700/746] loss 5.63, ppl 277.95, throughput 1217.44 samples/s\n",
"[Epoch 1] throughput 1206.55 samples/s\n",
"[Epoch 1] time cost 54.37s, valid loss 5.49, valid ppl 241.20\n",
"test loss 5.40, test ppl 221.14\n",
"[Epoch 2 Batch 100/746] loss 5.61, ppl 273.02, throughput 1188.24 samples/s\n",
"[Epoch 2 Batch 200/746] loss 5.53, ppl 252.61, throughput 1202.37 samples/s\n",
"[Epoch 2 Batch 300/746] loss 5.50, ppl 245.29, throughput 1200.85 samples/s\n",
"[Epoch 2 Batch 400/746] loss 5.50, ppl 244.17, throughput 1202.60 samples/s\n",
"[Epoch 2 Batch 500/746] loss 5.40, ppl 222.01, throughput 1213.11 samples/s\n",
"[Epoch 2 Batch 600/746] loss 5.34, ppl 208.07, throughput 1208.37 samples/s\n",
"[Epoch 2 Batch 700/746] loss 5.31, ppl 203.18, throughput 1208.56 samples/s\n",
"[Epoch 2] throughput 1205.63 samples/s\n",
"[Epoch 2] time cost 54.39s, valid loss 5.25, valid ppl 190.18\n",
"test loss 5.16, test ppl 174.20\n",
"Total training throughput 990.11 samples/s\n"
]
}
],
"source": [
"train(model, train_data, val_data, test_data, epochs)"
"train(model, train_data, val_data, test_data, epochs, lr)"
]
},
{
Expand All @@ -377,13 +379,13 @@
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 4982k 100 4982k 0 0 6107k 0 --:--:-- --:--:-- --:--:-- 6105k\n",
"100 4982k 100 4982k 0 0 29.1M 0 --:--:-- --:--:-- --:--:-- 28.9M\n",
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 390k 100 390k 0 0 1052k 0 --:--:-- --:--:-- --:--:-- 1055k\n",
"100 390k 100 390k 0 0 5205k 0 --:--:-- --:--:-- --:--:-- 5205k\n",
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 439k 100 439k 0 0 1195k 0 --:--:-- --:--:-- --:--:-- 1194k\n",
"100 439k 100 439k 0 0 4724k 0 --:--:-- --:--:-- --:--:-- 4724k\n",
"['ptb.test.txt', 'ptb.train.txt', 'ptb.valid.txt']\n"
]
}
Expand Down Expand Up @@ -419,7 +421,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Best validation loss 6.36, test ppl 575.78\n"
"Best validation loss 6.49, test ppl 660.12\n"
]
}
],
Expand All @@ -437,24 +439,21 @@
"name": "stdout",
"output_type": "stream",
"text": [
"[Epoch 0] throughput 1714.99 samples/s\n",
"[Epoch 0] time cost 2.41s, valid loss 5.32, valid ppl 205.25\n",
"test loss 5.32, test ppl 205.25\n",
"[Epoch 1] throughput 1682.03 samples/s\n",
"[Epoch 1] time cost 2.41s, valid loss 5.04, valid ppl 154.75\n",
"test loss 5.04, test ppl 154.75\n",
"[Epoch 2] throughput 1979.00 samples/s\n",
"[Epoch 2] time cost 2.42s, valid loss 4.96, valid ppl 143.20\n",
"test loss 4.96, test ppl 143.20\n",
"Total training throughput 620.79 samples/s\n"
"[Epoch 0] throughput 1183.07 samples/s\n",
"[Epoch 0] time cost 3.79s, valid loss 5.19, valid ppl 179.67\n",
"test loss 5.19, test ppl 179.67\n",
"[Epoch 1] throughput 1225.89 samples/s\n",
"[Epoch 1] time cost 3.71s, valid loss 5.30, valid ppl 200.73\n",
"Learning rate now 5.000000\n",
"[Epoch 2] throughput 1207.51 samples/s\n",
"[Epoch 2] time cost 3.75s, valid loss 4.70, valid ppl 110.38\n",
"test loss 4.70, test ppl 110.38\n",
"Total training throughput 456.48 samples/s\n"
]
}
],
"source": [
"lr = 20\n",
"epochs = 5\n",
"\n",
"train(model, ptb_val_data, ptb_val_data, ptb_val_data, 3)"
"train(model, ptb_val_data, ptb_val_data, ptb_val_data, epochs=3, lr=20)"
]
},
{
Expand All @@ -477,21 +476,21 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 2",
"display_name": "Python 3",
"language": "python",
"name": "python2"
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.12"
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
Expand Down
2 changes: 1 addition & 1 deletion docs/examples/word_embedding/word_embedding.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"To attach word embedding to indexed words in `vocab`, let us go on to create a fastText word embedding instance by specifying the embedding name `fasttext` and the source name `wiki.simple.vec`."
"To attach word embedding to indexed words in `vocab`, let us go on to create a fastText word embedding instance by specifying the embedding name `fasttext` and the source name `wiki.simple`."
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion docs/examples/word_embedding_evaluation/word_embedding_evaluation.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,7 +479,7 @@
"pd.options.display.max_rows = 999\n",
"pd.options.display.precision = 3\n",
"\n",
"df = pd.read_table(\"../../scripts/word_embedding_evaluation/results-vocablimit.csv\",\n",
"df = pd.read_table(\"../../../scripts/word_embedding_evaluation/results-vocablimit.csv\",\n",
" header=None, names=[\n",
" \"evaluation_type\", \"dataset\", \"kwargs\", \"embedding_name\",\n",
" \"embedding_source\", \"evaluation\", \"value\", \"num_samples\"\n",
Expand Down

0 comments on commit f9edbfe

Please sign in to comment.