make_dataset_emb.py

#!/usr/bin/env python
from pandas.core.arrays import boolean
import config as cfg
from tools import file_reader, file_writer, feature_maker
from tools import preprocessor, neural_embedder
from utility import embedder
import pandas as pd
import numpy as np
from pathlib import Path
from sklearn.decomposition import PCA
import tensorflow as tf
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, precision_score
from sklearn.metrics import recall_score, roc_auc_score

USE_CROSS_VALID = True
ENABLE_EMB_VIZ = False
VERBOSE = False

def main():
    make_complete_emb()
    make_compliance_emb()
    make_fall_emb()
    make_fall_test_emb()
    
def get_config(df_to_enc, target_name, artifacts_path):
    return {
        "df": df_to_enc,
        "target_name": target_name,
        "train_ratio": 0.8,
        "network_layers": ([128]),
        "epochs": 200,
        "batch_size": 32,
        "verbose": VERBOSE,
        "artifacts_path": artifacts_path
    }
    
def make_complete_emb():
    target_name = 'Complete'
    ats = {str(i)+'Ats':str for i in range(1, cfg.ATS_RESOLUTION+1)}
    df = file_reader.read_csv(cfg.PROCESSED_DATA_DIR,
                              'complete.csv',
                              converters=ats)
    
    emb_cols = df.filter(regex='((\d+)[Ats])\w+', axis=1)
    n_numerical_cols = df.shape[1] - emb_cols.shape[1] - 1
    
    df_to_enc = df.iloc[:,n_numerical_cols:]
    artifacts_path = cfg.COMPLIANCE_EMB_DIR
    
    if USE_CROSS_VALID:
        df_enc = encode_dataframe_cv(df_to_enc, target_name, artifacts_path)
    else:
        df_enc = encode_dataframe(df_to_enc, target_name, artifacts_path)
    
    ats_cols = [str(i)+'Ats' for i in range(1, cfg.ATS_RESOLUTION+1)]
    df = df.drop(ats_cols, axis=1)
    df = pd.concat([df.drop(target_name, axis=1), df_enc, df.pop(target_name)], axis=1)
    
    if USE_CROSS_VALID:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'complete_emb_cv.csv')
    else:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'complete_emb.csv')
    
def make_compliance_emb():
    target_name = 'Compliance'
    ats = {str(i)+'Ats':str for i in range(1, cfg.ATS_RESOLUTION+1)}
    df = file_reader.read_csv(cfg.PROCESSED_DATA_DIR,
                              f'compliance.csv',
                              converters=ats)
    
    emb_cols = df.filter(regex='((\d+)[Ats])\w+', axis=1)
    n_numerical_cols = df.shape[1] - emb_cols.shape[1] - 1
    
    df_to_enc = df.iloc[:,n_numerical_cols:]
    artifacts_path = cfg.COMPLIANCE_EMB_DIR
    
    if USE_CROSS_VALID:
        df_enc = encode_dataframe_cv(df_to_enc, target_name, artifacts_path)
    else:
        df_enc = encode_dataframe(df_to_enc, target_name, artifacts_path)
    
    ats_cols = [str(i)+'Ats' for i in range(1, cfg.ATS_RESOLUTION+1)]
    df = df.drop(ats_cols, axis=1)
    df = pd.concat([df.drop(target_name, axis=1), df_enc, df.pop(target_name)], axis=1)
    
    if USE_CROSS_VALID:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'compliance_emb_cv.csv')
    else:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'compliance_emb.csv')
    
def make_fall_emb():
    target_name = 'Fall'
    ats = {str(i)+'Ats':str for i in range(1, cfg.ATS_RESOLUTION+1)}
    df = file_reader.read_csv(cfg.PROCESSED_DATA_DIR,
                              f'fall.csv',
                              converters=ats)
    
    emb_cols = df.filter(regex='((\d+)[Ats])\w+', axis=1)
    n_numerical_cols = df.shape[1] - emb_cols.shape[1] - 1
    
    df_to_enc = df.iloc[:,n_numerical_cols:]
    artifacts_path = cfg.FALL_EMB_DIR
    
    if USE_CROSS_VALID:
        df_enc = encode_dataframe_cv(df_to_enc, target_name, artifacts_path)
    else:
        df_enc = encode_dataframe(df_to_enc, target_name, artifacts_path)
    
    ats_cols = [str(i)+'Ats' for i in range(1, cfg.ATS_RESOLUTION+1)]
    df = df.drop(ats_cols, axis=1)
    df = pd.concat([df.drop(target_name, axis=1), df_enc, df.pop(target_name)], axis=1)
    
    if USE_CROSS_VALID:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'fall_emb_cv.csv')
    else:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'fall_emb.csv')
    
def make_fall_test_emb():
    ex = {str(i)+'Ex':str for i in range(1, cfg.EX_RESOLUTION+1)}
    ats = {str(i)+'Ats':str for i in range(1, cfg.ATS_RESOLUTION+1)}
    converters = {**ex, **ats}
    df = file_reader.read_csv(cfg.PROCESSED_DATA_DIR,
                              f'fall_test.csv',
                              converters=converters)
    
    ats_cols = [str(i)+'Ats' for i in range(1, cfg.ATS_RESOLUTION+1)]
    ex_cols = [str(i)+'Ex' for i in range(1, cfg.EX_RESOLUTION+1)]

    df_ats_to_enc = df.filter(regex=f'Fall|((\d+)[Ats])\w+', axis=1)
    df_ats_to_enc = df_ats_to_enc.drop(['NumberFalls'], axis=1)
    
    df_ex_to_enc = df.filter(regex=f'Fall|((\d+)[Ex])\w+', axis=1)
    df_ex_to_enc = df_ex_to_enc.drop(['NumberFalls'], axis=1)

    artifacts_path = cfg.FALL_TEST_EMB_DIR
    
    if USE_CROSS_VALID:
        ats_enc = encode_dataframe_cv(df_ats_to_enc, 'Fall', artifacts_path)
        ex_enc = encode_dataframe_cv(df_ex_to_enc, 'Fall', artifacts_path)
    else:
        ats_enc = encode_dataframe(df_ats_to_enc, 'Fall', artifacts_path)
        ex_enc = encode_dataframe(df_ex_to_enc, 'Fall', artifacts_path)
    
    df = df.drop(ats_cols + ex_cols, axis=1)
    df = pd.concat([df.drop('Fall', axis=1), ats_enc, ex_enc, df.pop('Fall')], axis=1)
        
    if USE_CROSS_VALID:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'fall_test_emb_cv.csv')
    else:
        file_writer.write_csv(df, cfg.PROCESSED_DATA_DIR, 'fall_test_emb.csv')

def encode_dataframe_cv(df_to_enc, target_name, artifacts_path):
    params = get_config(df_to_enc, target_name, artifacts_path)
        
    X, y = preprocessor.get_X_y(df_to_enc, target_name)
    X, labels = preprocessor.encode_vector_label(X)
    y = np.array(y)
    
    network = neural_embedder.NeuralEmbedder(**params)
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
    callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                                mode='min',
                                                patience=3,
                                                verbose=1)
    
    weights = []
    for train_index, valid_index in skf.split(X, y):
        X_train, X_valid = X[train_index,:], X[valid_index,:]
        y_train, y_valid = y[train_index], y[valid_index]
        
        _ = network.fit(X_train, y_train, X_valid, y_valid,
                        callbacks=[callback])
        embedded_weights = network.get_embedded_weights()
        weights.append(embedded_weights)
        
    new_weights = list()
    for weights_list_tuple in zip(*weights):
        new_weights.append(
            [np.array(weights_).mean(axis=0)\
                for weights_ in zip(*weights_list_tuple)])
    
    network.save_weights(new_weights)
    network.save_labels(labels)
        
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df_to_enc.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1] - 1):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = new_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(cfg.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

def encode_dataframe(df_to_enc, target_name, artifacts_path):
    params = get_config(df_to_enc, target_name, artifacts_path)
        
    X_train, X_val, y_train, y_val, labels = preprocessor.prepare_data_for_embedder(df_to_enc,
                                                                                    target_name,
                                                                                    params['train_ratio'])
    
    network = neural_embedder.NeuralEmbedder(**params)
    network.fit(X_train, y_train, X_val, y_val)
    network.save_model()
    embedded_weights = network.get_embedded_weights()
    network.save_weights(embedded_weights)
    network.save_labels(labels)
    
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df_to_enc.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1] - 1):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = embedded_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(cfg.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

if __name__ == "__main__":
    main()