make_dataset_emb.py

#!/usr/bin/env python
from tools import file_reader, file_writer
from tools import preprocessor, neural_embedder
import pandas as pd
import numpy as np
import paths as pt
from pathlib import Path
from sklearn.decomposition import PCA
import tensorflow as tf
from sklearn.model_selection import StratifiedKFold
import yaml

USE_CROSS_VALID = False
USE_GROUPING = False
ENABLE_EMB_VIZ = False

def main(ats_resolution: int = None):
    with open(Path.joinpath(pt.CONFIGS_DIR, "settings.yaml"), 'r') as stream:
        settings = yaml.safe_load(stream)
    
    if ats_resolution == None:
        ats_resolution = settings['ats_resolution']
    
    for target_name in ["Complete", "Compliance", "Fall"]:
        ats = {str(i)+'Ats':str for i in range(1, ats_resolution+1)}
        df = file_reader.read_csv(pt.PROCESSED_DATA_DIR,
                                f'{target_name.lower()}.csv',
                                converters=ats)
    
        # Make a df to be encoded
        emb_cols = df.filter(regex='((\d+)[Ats])\w+', axis=1)
        n_numerical_cols = df.shape[1] - emb_cols.shape[1] - 1
        df_to_enc = df.iloc[:,n_numerical_cols:]
        
        # Remove old columns from original df
        ats_cols = [str(i)+'Ats' for i in range(1, ats_resolution+1)]
        df = df.drop(ats_cols, axis=1)

        # Load embedded config
        with open(Path.joinpath(pt.CONFIGS_DIR,
                                f"{target_name.lower()}_emb.yaml"), 'r') as stream:
            emb_cfg = yaml.safe_load(stream)
    
        # Encode dataframe given params
        model_path = Path.joinpath(pt.ROOT_DIR, emb_cfg['model_path'])
        df_enc = encode_dataframe(df=df_to_enc,
                                  target_name=emb_cfg['target_name'],                              
                                  batch_size=emb_cfg['batch_size'],
                                  train_ratio=emb_cfg['train_ratio'],
                                  epochs=emb_cfg['num_epochs'],
                                  optimizer=emb_cfg['optimizer'],
                                  network_layers=emb_cfg['network_layers'],
                                  verbose=emb_cfg['verbose'],
                                  model_path=model_path)
    
        df_rand = pd.DataFrame(np.random.rand(len(df),1), columns=['Rand'])
        df = pd.concat([df.drop(target_name, axis=1), df_rand, df_enc, df.pop(target_name)], axis=1)
        file_writer.write_csv(df, pt.PROCESSED_DATA_DIR, f'{target_name.lower()}_emb.csv')
        
def make_fall_test_emb(ats_resolution):
    ex = {str(i)+'Ex':str for i in range(1, pt.EX_RESOLUTION+1)}
    ats = {str(i)+'Ats':str for i in range(1, ats_resolution+1)}
    converters = {**ex, **ats}
    df = file_reader.read_csv(pt.PROCESSED_DATA_DIR,
                              f'fall_test.csv',
                              converters=converters)
    
    ats_cols = [str(i)+'Ats' for i in range(1, ats_resolution+1)]
    ex_cols = [str(i)+'Ex' for i in range(1, pt.EX_RESOLUTION+1)]

    df_ats_to_enc = df.filter(regex=f'Fall|((\d+)[Ats])\w+', axis=1)
    df_ats_to_enc = df_ats_to_enc.drop(['NumberFalls'], axis=1)
    
    df_ex_to_enc = df.filter(regex=f'Fall|((\d+)[Ex])\w+', axis=1)
    df_ex_to_enc = df_ex_to_enc.drop(['NumberFalls'], axis=1)

    artifacts_path = pt.FALL_TEST_EMB_DIR
    ats_enc = encode_dataframe(df_ats_to_enc, 'Fall', artifacts_path)
    ex_enc = encode_dataframe(df_ex_to_enc, 'Fall', artifacts_path)
    
    df = df.drop(ats_cols + ex_cols, axis=1)
    df = pd.concat([df.drop('Fall', axis=1), ats_enc, ex_enc, df.pop('Fall')], axis=1)
    file_writer.write_csv(df, pt.PROCESSED_DATA_DIR, 'fall_test_emb.csv')

def encode_dataframe(df, target_name, batch_size, train_ratio, epochs,
                     optimizer, network_layers, verbose, model_path):
    X_train, X_val, y_train, y_val, labels = preprocessor.prepare_data_for_embedder(df,
                                                                                    target_name,
                                                                                    train_ratio)
    network = neural_embedder.NeuralEmbedder(df=df, target_name=target_name, epochs=epochs,
                                             batch_size=batch_size, network_layers=network_layers,
                                             optimizer_fn=optimizer, verbose=verbose, model_path=model_path)
    network.fit(X_train, y_train, X_val, y_val)
    network.save_model()
    embedded_weights = network.get_embedded_weights()
    network.save_weights(embedded_weights)
    network.save_labels(labels)
    
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1]):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = embedded_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(pt.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

def encode_dataframe_cv(df, target_name, batch_size, train_ratio,
                        epochs, network_layers, verbose, model_path):        
    X, y = preprocessor.get_X_y(df, target_name)
    X, labels = preprocessor.encode_vector_label(X)
    y = np.array(y)

    network = neural_embedder.NeuralEmbedder(df=df, target_name=target_name, epochs=epochs,
                                             batch_size=batch_size, network_layers=network_layers,
                                             verbose=verbose, model_path=model_path)
    
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
    es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                                   mode='min',
                                                   patience=3,
                                                   verbose=0)
    
    weights = []
    for train_index, valid_index in skf.split(X, y):
        X_train, X_valid = X[train_index,:], X[valid_index,:]
        y_train, y_valid = y[train_index], y[valid_index]
        
        _ = network.fit(X_train, y_train, X_valid, y_valid,
                        callbacks=[es_callback])
        embedded_weights = network.get_embedded_weights()
        weights.append(embedded_weights)
        
    new_weights = list()
    for weights_list_tuple in zip(*weights):
        new_weights.append(
            [np.array(weights_).mean(axis=0)\
                for weights_ in zip(*weights_list_tuple)])
    
    network.save_weights(new_weights)
    network.save_labels(labels)
        
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1]):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = new_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(pt.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

if __name__ == "__main__":
    main()