util.py

import itertools
from scipy import spatial
import os
import pickle
import string
import numpy as np
from nltk import tokenize
from sklearn.model_selection import train_test_split
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing.text import Tokenizer


def cosine_similarity(a, b):
    return 1 - spatial.distance.cosine(a, b)


def compute_pairwise_cosine_sim(tok_vecs):
    pairs = list(itertools.combinations(tok_vecs, 2))
    cos_sim = []
    for pair in pairs:
        sim = cosine_similarity(pair[0], pair[1])
        cos_sim.append(sim)
    return cos_sim


def read_bert_vectors(word, bert_dump_dir):
    word_clean = word.translate(str.maketrans('', '', string.punctuation))
    if os.path.isdir(os.path.join(bert_dump_dir, word_clean)):
        word_dir = os.path.join(bert_dump_dir, word_clean)
    elif os.path.isdir(os.path.join(bert_dump_dir, word)):
        word_dir = os.path.join(bert_dump_dir, word)
    else:
        raise Exception(word + " not found")
    filepaths = [os.path.join(word_dir, o) for o in os.listdir(word_dir) if
                 os.path.isfile(os.path.join(word_dir, o))]
    tok_vecs = []
    for path in filepaths:
        try:
            with open(path, "rb") as input_file:
                vec = pickle.load(input_file)
            tok_vecs.append(vec)
        except Exception as e:
            print("Exception while reading BERT pickle file: ", path, e)
    return tok_vecs


def get_relevant_dirs(bert_dump_dir):
    print("Getting relevant dirs..")
    dirs = os.listdir(bert_dump_dir)
    dir_dict = {}
    for dir in dirs:
        dir_dict[dir] = 1

    print("Dir dict ready..")
    dir_set = set()
    for i, dir in enumerate(dirs):
        if i % 1000 == 0:
            print("Finished checking dirs: " + str(i) + " out of: " + str(len(dirs)))
        dir_new = dir.translate(str.maketrans('', '', string.punctuation))
        if len(dir_new) == 0:
            continue
        try:
            temp = dir_dict[dir_new]
            dir_set.add(dir_new)
        except:
            dir_set.add(dir)
    return dir_set


def to_tokenized_string(sentence):
    tokenized = " ".join([t.text for t in sentence.tokens])
    return tokenized


def create_label_index_maps(labels):
    label_to_index = {}
    index_to_label = {}
    for i, label in enumerate(labels):
        label_to_index[label] = i
        index_to_label[i] = label
    return label_to_index, index_to_label


def make_one_hot(y, label_to_index):
    labels = list(label_to_index.keys())
    n_classes = len(labels)
    y_new = []
    for label in y:
        current = np.zeros(n_classes)
        i = label_to_index[label]
        current[i] = 1.0
        y_new.append(current)
    y_new = np.asarray(y_new)
    return y_new


def prep_data(max_sentence_length, max_sentences, texts, tokenizer):
    data = np.zeros((len(texts), max_sentences, max_sentence_length), dtype='int32')
    documents = []
    for text in texts:
        sents = tokenize.sent_tokenize(text)
        documents.append(sents)
    for i, sentences in enumerate(documents):
        tokenized_sentences = tokenizer.texts_to_sequences(
            sentences
        )
        tokenized_sentences = pad_sequences(
            tokenized_sentences, maxlen=max_sentence_length
        )

        pad_size = max_sentences - tokenized_sentences.shape[0]

        if pad_size < 0:
            tokenized_sentences = tokenized_sentences[0:max_sentences]
        else:
            tokenized_sentences = np.pad(
                tokenized_sentences, ((0, pad_size), (0, 0)),
                mode='constant', constant_values=0
            )

        data[i] = tokenized_sentences[None, ...]
    return data


def create_train_dev(texts, labels, tokenizer, max_sentences=15, max_sentence_length=100, max_words=20000):
    data = prep_data(max_sentence_length, max_sentences, texts, tokenizer)
    X_train, X_test, y_train, y_test = train_test_split(data, labels, test_size=0.1, random_state=42)
    return X_train, y_train, X_test, y_test


def get_from_one_hot(pred, index_to_label):
    pred_labels = np.argmax(pred, axis=-1)
    ans = []
    for l in pred_labels:
        ans.append(index_to_label[l])
    return ans


def calculate_df_doc_freq(df):
    docfreq = {}
    docfreq["UNK"] = len(df)
    for index, row in df.iterrows():
        line = row["sentence"]
        words = line.strip().split()
        temp_set = set(words)
        for w in temp_set:
            try:
                docfreq[w] += 1
            except:
                docfreq[w] = 1
    return docfreq


def calculate_doc_freq(docs):
    docfreq = {}
    for doc in docs:
        words = doc.strip().split()
        temp_set = set(words)
        for w in temp_set:
            try:
                docfreq[w] += 1
            except:
                docfreq[w] = 1
    return docfreq


def calculate_inv_doc_freq(df, docfreq):
    inv_docfreq = {}
    N = len(df)
    for word in docfreq:
        inv_docfreq[word] = np.log(N / docfreq[word])
    return inv_docfreq


def create_word_index_maps(word_vec):
    word_to_index = {}
    index_to_word = {}
    words = list(word_vec.keys())
    for i, word in enumerate(words):
        word_to_index[word] = i
        index_to_word[i] = word
    return word_to_index, index_to_word


def get_vec(word, word_cluster, stop_words):
    if word in stop_words:
        return []
    t = word.split("$")
    if len(t) == 1:
        prefix = t[0]
        cluster = 0
    elif len(t) == 2:
        prefix = t[0]
        cluster = t[1]
        try:
            cluster = int(cluster)
        except:
            prefix = word
            cluster = 0
    else:
        prefix = "".join(t[:-1])
        cluster = t[-1]
        try:
            cluster = int(cluster)
        except:
            cluster = 0

    word_clean = prefix.translate(str.maketrans('', '', string.punctuation))
    if len(word_clean) == 0 or word_clean in stop_words:
        return []
    try:
        vec = word_cluster[word_clean][cluster]
    except:
        try:
            vec = word_cluster[prefix][cluster]
        except:
            try:
                vec = word_cluster[word][0]
            except:
                vec = []
    return vec


def get_label_docs_dict(df, label_term_dict, pred_labels):
    label_docs_dict = {}
    for l in label_term_dict:
        label_docs_dict[l] = []
    for index, row in df.iterrows():
        line = row["sentence"]
        label_docs_dict[pred_labels[index]].append(line)
    return label_docs_dict


def add_all_interpretations(label_term_dict, word_cluster):
    print("Considering all interpretations of seed words..")
    new_dic = {}
    for l in label_term_dict:
        for word in label_term_dict[l]:
            try:
                cc = word_cluster[word]
                n_inter = len(cc)
            except:
                continue

            if n_inter == 1:
                try:
                    new_dic[l].append(word)
                except:
                    new_dic[l] = [word]
            else:
                for i in range(n_inter):
                    con_word = word + "$" + str(i)
                    try:
                        new_dic[l].append(con_word)
                    except:
                        new_dic[l] = [con_word]
    return new_dic


def print_label_term_dict(label_term_dict, components, print_components=True):
    for label in label_term_dict:
        print(label)
        print("*" * 80)
        for val in label_term_dict[label]:
            try:
                if print_components:
                    print(val, components[label][val])
                else:
                    print(val)
            except Exception as e:
                print("Exception occurred: ", e, val)


def fit_get_tokenizer(data, max_words):
    tokenizer = Tokenizer(num_words=max_words, filters='!"#%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n')
    tokenizer.fit_on_texts(data)
    return tokenizer