Python32-Python实现word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 简介\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Genism是一个开源的Python库，用于便捷高效地提取文档中的语义话题。它用于处理原始的、非结构化的电子文本（“纯文本”），gensim中的一些算法，如 Latent Semantic Analysis（潜在语义分析）、 Latent Dirichlet Allocation（潜在Dirichlet分布）、Random Projections（随机预测）通过检查训练文档中的共现实体来挖掘语义结构。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 快速上手"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "2018-05-09 15:06:25,490:INFO:'pattern' package not found; tag filters are not available for English\n"
     ]
    }
   ],
   "source": [
    "#创建一个小的语料库\n",
    "from gensim import corpora,models,similarities\n",
    "\n",
    "corpus=[[(0,1.0),(1,1.0),(2,1.0)],\n",
    "        [(2, 1.0), (3, 1.0), (4, 1.0), (5, 1.0), (6, 1.0), (8, 1.0)],\n",
    "        [(1, 1.0), (3, 1.0), (4, 1.0), (7, 1.0)],\n",
    "        [(0, 1.0), (4, 2.0), (7, 1.0)],\n",
    "        [(3, 1.0), (5, 1.0), (6, 1.0)],\n",
    "        [(9, 1.0)],\n",
    "        [(9, 1.0), (10, 1.0)],\n",
    "        [(9, 1.0), (10, 1.0), (11, 1.0)],\n",
    "        [(8, 1.0), (10, 1.0), (11, 1.0)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-09 15:06:25,500:INFO:collecting document frequencies\n",
      "2018-05-09 15:06:25,502:INFO:PROGRESS: processing document #0\n",
      "2018-05-09 15:06:25,504:INFO:calculating IDF weights for 9 documents and 11 features (28 matrix non-zeros)\n"
     ]
    }
   ],
   "source": [
    "#对向量进行加权\n",
    "tfidf=models.TfidfModel(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.8075244024440723), (4, 0.5898341626740045)]\n"
     ]
    }
   ],
   "source": [
    "vec=[(0,1),(4,1)]\n",
    "print(tfidf[vec])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-09 15:06:25,726:INFO:creating sparse index\n",
      "2018-05-09 15:06:25,727:INFO:creating sparse matrix from corpus\n",
      "2018-05-09 15:06:25,728:INFO:PROGRESS: at document #0\n",
      "2018-05-09 15:06:25,729:INFO:created <9x12 sparse matrix of type '<class 'numpy.float32'>'\n",
      "\twith 28 stored elements in Compressed Sparse Row format>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.4662244), (1, 0.19139354), (2, 0.2460055), (3, 0.82094586), (4, 0.0), (5, 0.0), (6, 0.0), (7, 0.0), (8, 0.0)]\n"
     ]
    }
   ],
   "source": [
    "index= similarities.SparseMatrixSimilarity(tfidf[corpus],num_features=12)\n",
    "sims=index[tfidf[vec]]\n",
    "print(list(enumerate(sims)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 生成中文语料的word2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 对中文语料进行分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "2018-05-09 15:06:26,133:DEBUG:Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache C:\\Users\\Yanqiang\\AppData\\Local\\Temp\\jieba.cache\n",
      "2018-05-09 15:06:27,277:DEBUG:Dumping model to file cache C:\\Users\\Yanqiang\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 1.236 seconds.\n",
      "2018-05-09 15:06:27,370:DEBUG:Loading model cost 1.236 seconds.\n",
      "Prefix dict has been built succesfully.\n",
      "2018-05-09 15:06:27,371:DEBUG:Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import jieba\n",
    "sentences_file=open(\"files/data/python32-data/sentence.txt\",encoding='utf8')\n",
    "word_file=open(\"files/data/python32-word.txt\",\"a\",encoding=\"utf8\")\n",
    "lines=sentences_file.readlines()\n",
    "for line in lines:\n",
    "    line.replace('\\t','').replace('\\n','').replace(' ','')\n",
    "    segment_words=jieba.cut(line,cut_all=False)\n",
    "    word_file.write(\" \".join(segment_words))\n",
    "sentences_file.close()\n",
    "word_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 使用gensim的word2vec训练模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "参考：[python初步实现word2vec](http://blog.csdn.net/xiaoquantouer/article/details/53583980)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-09 15:07:52,206:INFO:collecting all words and their counts\n",
      "2018-05-09 15:07:52,208:INFO:PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2018-05-09 15:07:52,214:INFO:collected 4152 word types from a corpus of 20740 raw words and 3 sentences\n",
      "2018-05-09 15:07:52,215:INFO:Loading a fresh vocabulary\n",
      "2018-05-09 15:07:52,218:INFO:min_count=5 retains 579 unique words (13% of original 4152, drops 3573)\n",
      "2018-05-09 15:07:52,219:INFO:min_count=5 leaves 15399 word corpus (74% of original 20740, drops 5341)\n",
      "2018-05-09 15:07:52,223:INFO:deleting the raw counts dictionary of 4152 items\n",
      "2018-05-09 15:07:52,224:INFO:sample=0.001 downsamples 73 most-common words\n",
      "2018-05-09 15:07:52,224:INFO:downsampling leaves estimated 9696 word corpus (63.0% of prior 15399)\n",
      "2018-05-09 15:07:52,226:INFO:estimated required memory for 579 words and 200 dimensions: 1215900 bytes\n",
      "2018-05-09 15:07:52,227:INFO:resetting layer weights\n",
      "2018-05-09 15:07:52,242:INFO:training model with 3 workers on 579 vocabulary and 200 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "2018-05-09 15:07:52,257:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:07:52,260:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:07:52,264:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:07:52,265:INFO:EPOCH - 1 : training on 20740 raw words (9639 effective words) took 0.0s, 502553 effective words/s\n",
      "2018-05-09 15:07:52,280:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:07:52,285:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:07:52,287:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:07:52,288:INFO:EPOCH - 2 : training on 20740 raw words (9688 effective words) took 0.0s, 571971 effective words/s\n",
      "2018-05-09 15:07:52,301:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:07:52,306:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:07:52,308:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:07:52,308:INFO:EPOCH - 3 : training on 20740 raw words (9774 effective words) took 0.0s, 577347 effective words/s\n",
      "2018-05-09 15:07:52,322:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:07:52,330:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:07:52,332:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:07:52,333:INFO:EPOCH - 4 : training on 20740 raw words (9720 effective words) took 0.0s, 477067 effective words/s\n",
      "2018-05-09 15:07:52,348:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:07:52,354:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:07:52,359:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:07:52,362:INFO:EPOCH - 5 : training on 20740 raw words (9631 effective words) took 0.0s, 421856 effective words/s\n",
      "2018-05-09 15:07:52,364:INFO:training on a 103700 raw words (48452 effective words) took 0.1s, 397561 effective words/s\n",
      "2018-05-09 15:07:52,366:WARNING:under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:13: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).\n",
      "  del sys.path[0]\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "2018-05-09 15:07:52,371:INFO:precomputing L2-norms of word weight vectors\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:27: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "E:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:33: DeprecationWarning: Call to deprecated `doesnt_match` (Method will be removed in 4.0.0, use self.wv.doesnt_match() instead).\n",
      "2018-05-09 15:07:52,433:INFO:saving Word2Vec object under files/data/python32-data/企业关系.model, separately None\n",
      "2018-05-09 15:07:52,435:INFO:not storing attribute vectors_norm\n",
      "2018-05-09 15:07:52,436:INFO:not storing attribute cum_table\n",
      "2018-05-09 15:07:52,478:INFO:saved files/data/python32-data/企业关系.model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输出模型 Word2Vec(vocab=579, size=200, alpha=0.025)\n",
      "【企业】和【公司】的相似度为：0.9999506965308783\n",
      "\n",
      "与【科技】最相关的词有：\n",
      "\n",
      "， 0.9999631643295288\n",
      "是 0.9999608993530273\n",
      "的 0.9999598264694214\n",
      "有限公司 0.9999590516090393\n",
      "产品 0.9999586343765259\n",
      "。 0.9999575614929199\n",
      "公司 0.9999558925628662\n",
      "成为 0.9999552965164185\n",
      "合作 0.9999547004699707\n",
      "合作伙伴 0.9999545216560364\n",
      "： 0.999954104423523\n",
      "核心 0.9999538660049438\n",
      "和 0.9999529123306274\n",
      "及 0.9999523162841797\n",
      "等 0.9999514818191528\n",
      "中国 0.9999513626098633\n",
      "经销商 0.9999507069587708\n",
      "在 0.9999505281448364\n",
      "代理商 0.9999504089355469\n",
      "供应商 0.9999495148658752\n",
      "*********\n",
      "\n",
      "公司-产品 生产\n",
      "及 0.9998089075088501\n",
      "月 0.999807596206665\n",
      "成为 0.9998065829277039\n",
      "*********\n",
      "\n",
      "不合群的词：企业\n",
      "***********\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 导入包\n",
    "from gensim.models import word2vec\n",
    "import logging\n",
    "\n",
    "#初始化\n",
    "logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)\n",
    "sentences=word2vec.Text8Corpus(\"files/data/python32-data/word.txt\")#加载分词语料\n",
    "model=word2vec.Word2Vec(sentences,size=200)#训练skip-gram模型，默认window=5\n",
    "print(\"输出模型\",model)\n",
    "\n",
    "#计算两个单词的相似度\n",
    "try:\n",
    "    y1=model.similarity(\"企业\",\"公司\")\n",
    "except KeyError:\n",
    "    y1=0\n",
    "print(\"【企业】和【公司】的相似度为：{}\\n\".format(y1))\n",
    "\n",
    "#/计算某个词的相关词列表\n",
    "y2=model.most_similar(\"科技\",topn=20)#20个最相关的\n",
    "print(\"与【科技】最相关的词有：\\n\")\n",
    "for word in y2:\n",
    "    print(word[0],word[1])\n",
    "print(\"*********\\n\")\n",
    "\n",
    "#寻找对应关系\n",
    "print(\"公司-产品\",\"生产\")\n",
    "y3=model.most_similar([\"公司\",\"产品\"],[\"生产\"],topn=3)\n",
    "for word in y3:\n",
    "    print(word[0],word[1])\n",
    "print(\"*********\\n\")\n",
    "\n",
    "#寻找不合群的词\n",
    "y4 =model.doesnt_match(u\"企业 公司 是 合作伙伴\".split())  \n",
    "print(\"不合群的词：{}\".format(y4))  \n",
    "print(\"***********\\n\"  )\n",
    "\n",
    "#保存模型\n",
    "model.save(\"files/data/python32-data/企业关系.model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4 生成英文语料的word2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.1 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "import re\n",
    "\n",
    "# 加载语料内容\n",
    "with open('files/data/python32-data/alice.txt',encoding='utf-8') as file:\n",
    "    txt_raw=file.read()\n",
    "# nltk    将段落分为句\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "sentences_raw = tokenizer.tokenize(txt_raw) # 句子列表\n",
    "# for sen in sentences_raw:\n",
    "#     print(sen)\n",
    "\n",
    "# 对句子列表进行预处理\n",
    "sntncs = []\n",
    "stops = set(nltk.corpus.stopwords.words('english'))\n",
    "for sntnc in sentences_raw:\n",
    "    lttr_only = re.sub('[^a-zA-z]', \" \", sntnc)# 取出标点符号与数字\n",
    "    wrds = lttr_only.lower().split() # 大写转为小写\n",
    "    wrds_mnng = [w for w in wrds if not w in stops] # 去除停用词\n",
    "    sntncs += [wrds_mnng]\n",
    "# print(sntncs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 使用genism训练word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2018-05-09 15:28:00,788:INFO:collecting all words and their counts\n",
      "2018-05-09 15:28:00,789:INFO:PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2018-05-09 15:28:00,793:INFO:collected 2866 word types from a corpus of 14027 raw words and 1744 sentences\n",
      "2018-05-09 15:28:00,794:INFO:Loading a fresh vocabulary\n",
      "2018-05-09 15:28:00,798:INFO:min_count=10 retains 300 unique words (10% of original 2866, drops 2566)\n",
      "2018-05-09 15:28:00,799:INFO:min_count=10 leaves 8052 word corpus (57% of original 14027, drops 5975)\n",
      "2018-05-09 15:28:00,801:INFO:deleting the raw counts dictionary of 2866 items\n",
      "2018-05-09 15:28:00,802:INFO:sample=0.001 downsamples 112 most-common words\n",
      "2018-05-09 15:28:00,803:INFO:downsampling leaves estimated 5563 word corpus (69.1% of prior 8052)\n",
      "2018-05-09 15:28:00,804:INFO:estimated required memory for 300 words and 1000 dimensions: 2550000 bytes\n",
      "2018-05-09 15:28:00,806:INFO:resetting layer weights\n",
      "2018-05-09 15:28:00,820:INFO:training model with 4 workers on 300 vocabulary and 1000 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
      "2018-05-09 15:28:00,827:INFO:worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-09 15:28:00,829:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:28:00,837:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:28:00,850:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:28:00,851:INFO:EPOCH - 1 : training on 14027 raw words (5523 effective words) took 0.0s, 216992 effective words/s\n",
      "2018-05-09 15:28:00,857:INFO:worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-09 15:28:00,858:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:28:00,869:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:28:00,884:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:28:00,884:INFO:EPOCH - 2 : training on 14027 raw words (5575 effective words) took 0.0s, 191280 effective words/s\n",
      "2018-05-09 15:28:00,890:INFO:worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-09 15:28:00,891:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:28:00,901:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:28:00,916:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:28:00,917:INFO:EPOCH - 3 : training on 14027 raw words (5581 effective words) took 0.0s, 193354 effective words/s\n",
      "2018-05-09 15:28:00,923:INFO:worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-09 15:28:00,925:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:28:00,935:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:28:00,946:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:28:00,947:INFO:EPOCH - 4 : training on 14027 raw words (5580 effective words) took 0.0s, 233501 effective words/s\n",
      "2018-05-09 15:28:00,955:INFO:worker thread finished; awaiting finish of 3 more threads\n",
      "2018-05-09 15:28:00,963:INFO:worker thread finished; awaiting finish of 2 more threads\n",
      "2018-05-09 15:28:00,967:INFO:worker thread finished; awaiting finish of 1 more threads\n",
      "2018-05-09 15:28:00,981:INFO:worker thread finished; awaiting finish of 0 more threads\n",
      "2018-05-09 15:28:00,982:INFO:EPOCH - 5 : training on 14027 raw words (5544 effective words) took 0.0s, 176423 effective words/s\n",
      "2018-05-09 15:28:00,983:INFO:training on a 70135 raw words (27803 effective words) took 0.2s, 171194 effective words/s\n",
      "2018-05-09 15:28:00,984:WARNING:under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
      "2018-05-09 15:28:00,984:INFO:saving Word2Vec object under files/data/python32-data/alice.model, separately None\n",
      "2018-05-09 15:28:00,985:INFO:not storing attribute vectors_norm\n",
      "2018-05-09 15:28:00,985:INFO:not storing attribute cum_table\n",
      "2018-05-09 15:28:01,014:INFO:saved files/data/python32-data/alice.model\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import word2vec\n",
    "num_features = 1000 #是指特征向量的维度，默认为100。大的size需要更多的训练数据,但是效果会更好. 推荐值为几十到几百。\n",
    "min_word_count = 10# 可以对字典做截断. 词频少于min_count次数的单词会被丢弃掉, 默认值为5\n",
    "num_workers = 4 #控制训练的并行数。\n",
    "window = 5\n",
    "# gensim函数库的Word2Vec的参数说明 https://blog.csdn.net/szlcw1/article/details/52751314\n",
    "model = word2vec.Word2Vec(sntncs, workers=num_workers, \\\n",
    "        size=num_features, min_count=min_word_count, \\\n",
    "        window=window)\n",
    "model.save('files/data/python32-data/alice.model')        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}