doc2vec_expe2.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"doc2vec_expe2.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyMW/xik83YibIyrOLLVXcXa"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"GeMkvsjt5DU_"},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","from tqdm import tqdm\n","import torch\n","import ast\n","import os\n","import pickle\n","import shutil\n","import random\n","\n","from scipy.spatial import distance"]},{"cell_type":"code","source":["from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n","from nltk.corpus import stopwords"],"metadata":{"id":"uXJZ-PB85Jwe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# read database and samples\n","df_database = pd.read_pickle(\"./df_database.pkl\")\n","df_sample = pd.read_pickle(\"./df_sample.pkl\")"],"metadata":{"id":"0H9Q000N5K0a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["sample_sources = list(df_sample['source'])\n","base_sources = list(df_database['source'])"],"metadata":{"id":"a-IwvDtN5L0V"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# nb of letters to consider in cim10 codes\n","nb_letter = 3\n","# dic where keys are sources and values are cim10 codes for this doc\n","dic_source_cim10 = {}\n","df_base_new = pd.concat([df_database,df_sample]).reset_index()\n","for i in range(len(df_base_new)):\n","    source = df_base_new.loc[i,'source']\n","    cim10s = ast.literal_eval(df_base_new.loc[i,'list_cim10'])\n","    # cim10s = list(set([x.split(':')[-1][:nb_letter] for x in cim10s])) # at least one same DP/DAS\n","    # cim10s = list(set([x.split(':')[-1][:nb_letter] for x in cim10s if x.startswith('DP')])) # at least one same DP\n","    cim10s = list(set([x.split(':')[-1][:nb_letter] for x in cim10s if x.startswith('DAS')])) # at least one same DAS\n","    dic_source_cim10[source] = cim10s"],"metadata":{"id":"dlhvvFts5Myr"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# doc2vec\n","use doc2vec to find top k candidates for each sample in database, see how many of these candidates have at least one same DP/DAS as the sample itself."],"metadata":{"id":"Piga3NDj5QC4"}},{"cell_type":"code","source":["dim = 300 # dimension of vectors\n","dm = 0 # (0 for dbow, 1 for dm)\n","stop_words = set(stopwords.words('french'))"],"metadata":{"id":"IrCLNh065N9h"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# preprocess sample docs: tokenization, delete stop words and non alphabetic words, lower case\n","sample_docs = [x.split() for x in list(df_sample['observation_blob'])]\n","sample_docs = [[l.lower() for l in x if l.isalpha() and l not in stop_words] for x in sample_docs]"],"metadata":{"id":"wV1KqlFx5TZa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# doc2vec for samples\n","tagged_data = [TaggedDocument(doc, [i]) for i, doc in enumerate(sample_docs)]\n","model = Doc2Vec(tagged_data, vector_size=dim, window=5,\n","                dm=dm, min_count=2, epochs=100, workers=10)"],"metadata":{"id":"0-8Oc-wd5UkA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["vector_samples = []\n","for doc in tqdm(sample_docs):\n","    vector_samples.append(model.infer_vector(doc))"],"metadata":{"id":"0ZD1iWCw5Vm0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# preprocess database docs: tokenization, delete stop words and non alphabetic words, lower case\n","base_docs = [str(x).split() for x in list(df_database['observation_blob'])]\n","base_docs = [[l.lower() for l in x if l.isalpha() and l not in stop_words] for x in base_docs]"],"metadata":{"id":"iEo2fHdN5Wmd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# doc2vec for database docs\n","tagged_data_base = [TaggedDocument(doc, [i]) for i, doc in enumerate(base_docs)]\n","model_base = Doc2Vec(tagged_data_base, vector_size=dim, window=5,\n","                dm=dm, min_count=2, epochs=100, workers=10)"],"metadata":{"id":"9cNqQzfo5YrD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["vector_base = []\n","for doc in tqdm(base_docs):\n","    vector_base.append(model_base.infer_vector(doc))"],"metadata":{"id":"Otc4yber5Z04"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# for each sample, find top nb candidates from database\n","nb = 100\n","vec_base = np.array(vector_base)\n","dic_sample_cands = {}\n","for i in tqdm(range(len(sample_sources))):\n","    vec_sample = np.array(vector_samples[i])\n","    sim = np.matmul(vec_sample,vec_base.T)\n","    idx = torch.topk(torch.tensor(sim), k=nb, sorted=True).indices\n","    cands = [base_sources[x] for x in idx]\n","    dic_sample_cands[sample_sources[i]] = cands"],"metadata":{"id":"jyShL4nF5bI2"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Evaluation"],"metadata":{"id":"Rj1NUiZh5cip"}},{"cell_type":"code","source":["accs = []\n","for sample in tqdm(dic_sample_cands):\n","    sample_cim10 = dic_source_cim10[sample]\n","    hit = 0\n","    for cand in dic_sample_cands[sample]:\n","        cand_cim10 = dic_source_cim10[cand]\n","        if len([x for x in cand_cim10 if x in sample_cim10])>0:\n","            hit+=1\n","    acc = hit/len(dic_sample_cands[sample])\n","    accs.append(acc)"],"metadata":{"id":"niRtyJlx5fCc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["np.mean(accs)"],"metadata":{"id":"-f4chafX5gX4"},"execution_count":null,"outputs":[]}]}