make_dataset_emb.py

#!/usr/bin/env python
from tools import file_reader, file_writer
from tools import preprocessor, neural_embedder
import pandas as pd
import numpy as np
import paths as pt
from pathlib import Path
from sklearn.decomposition import PCA
import tensorflow as tf
from sklearn.model_selection import StratifiedKFold
import yaml

USE_CROSS_VALID = False
USE_GROUPING = False
ENABLE_EMB_VIZ = False

def main(ats_resolution: int = None):        
    for label_name in ["Complete", "Compliance", "Alarm", "Fall"]:
        with open(Path.joinpath(pt.CONFIGS_DIR,
                                f'{label_name.lower()}_emb.yaml'), 'r') as stream:
            settings = yaml.safe_load(stream)
            
        if ats_resolution == None:
            ats_resolution = settings['ats_resolution']
        if label_name == "Fall":
            ex_resolution = settings['ex_resolution']
        
        if label_name in ["Complete", "Compliance", "Alarm"]:
            ats = {str(i)+'Ats':str for i in range(1, ats_resolution+1)}
            df = file_reader.read_csv(pt.PROCESSED_DATA_DIR,
                                    f'{label_name.lower()}.csv',
                                    converters=ats)
        else:
            ex = {str(i)+'Ex':str for i in range(1, ex_resolution+1)}
            ats = {str(i)+'Ats':str for i in range(1, ats_resolution+1)}
            converters = {**ex, **ats}
            df = file_reader.read_csv(pt.PROCESSED_DATA_DIR,
                                    f'{label_name.lower()}.csv',
                                    converters=converters)
        
        if label_name in ["Complete", "Compliance", "Alarm"]:
            emb_cols = df.filter(regex='((\d+)[Ats])\w+', axis=1)
            n_numerical_cols = df.shape[1] - emb_cols.shape[1] - 1
            df_to_enc = df.iloc[:,n_numerical_cols:] 
            ats_cols = [str(i)+'Ats' for i in range(1, ats_resolution+1)]
            df = df.drop(ats_cols, axis=1)
        else:
            ats_cols = [str(i)+'Ats' for i in range(1, ats_resolution+1)]
            ex_cols = [str(i)+'Ex' for i in range(1, ex_resolution+1)]
            df_ats_to_enc = df.filter(regex=f'Fall|((\d+)[Ats])\w+', axis=1)
            df_ex_to_enc = df.filter(regex=f'Fall|((\d+)[Ex])\w+', axis=1)
            df = df.drop(ats_cols + ex_cols, axis=1)
        
        # Load embedded config
        with open(Path.joinpath(pt.CONFIGS_DIR,
                                f"{label_name.lower()}_emb.yaml"), 'r') as stream:
            emb_cfg = yaml.safe_load(stream)
    
        # Encode dataframe given params
        model_path = Path.joinpath(pt.ROOT_DIR, emb_cfg['model_path'])
        if label_name in ["Complete", "Compliance", "Alarm"]:
            df_enc = encode_dataframe(df=df_to_enc,
                                    target_name=emb_cfg['target_name'],
                                    batch_size=emb_cfg['batch_size'],
                                    train_ratio=emb_cfg['train_ratio'],
                                    epochs=emb_cfg['num_epochs'],
                                    optimizer=emb_cfg['optimizer'],
                                    network_layers=emb_cfg['network_layers'],
                                    verbose=emb_cfg['verbose'],
                                    model_path=model_path)
        else:
            ats_enc = encode_dataframe(df=df_ats_to_enc,
                                    target_name=emb_cfg['target_name'],
                                    batch_size=emb_cfg['batch_size'],
                                    train_ratio=emb_cfg['train_ratio'],
                                    epochs=emb_cfg['num_epochs_ats'],
                                    optimizer=emb_cfg['optimizer'],
                                    network_layers=emb_cfg['network_layers'],
                                    verbose=emb_cfg['verbose'],
                                    model_path=model_path)
            ex_enc = encode_dataframe(df=df_ex_to_enc,
                                    target_name=emb_cfg['target_name'],
                                    batch_size=emb_cfg['batch_size'],
                                    train_ratio=emb_cfg['train_ratio'],
                                    epochs=emb_cfg['num_epochs_ex'],
                                    optimizer=emb_cfg['optimizer'],
                                    network_layers=emb_cfg['network_layers'],
                                    verbose=emb_cfg['verbose'],
                                    model_path=model_path)
    
        df_rand = pd.DataFrame(np.random.rand(len(df),1), columns=['Rand']) # add random var
        
        if label_name in ["Complete", "Compliance", "Alarm"]:
            df = pd.concat([df.drop(label_name, axis=1), df_rand, df_enc, df.pop(label_name)], axis=1)
        else:
            df = pd.concat([df.drop(label_name, axis=1), df_rand, ats_enc, ex_enc,
                            df.pop(label_name)], axis=1)

        file_writer.write_csv(df, pt.PROCESSED_DATA_DIR, f'{label_name.lower()}_emb.csv')
        
def encode_dataframe(df, target_name, batch_size, train_ratio, epochs,
                     optimizer, network_layers, verbose, model_path):
    X_train, X_val, y_train, y_val, labels = preprocessor.prepare_data_for_embedder(df,
                                                                                    target_name,
                                                                                    train_ratio)
    network = neural_embedder.NeuralEmbedder(df=df, target_name=target_name, epochs=epochs,
                                             batch_size=batch_size, network_layers=network_layers,
                                             optimizer_fn=optimizer, verbose=verbose, model_path=model_path)
    network.fit(X_train, y_train, X_val, y_val)
    network.save_model()
    embedded_weights = network.get_embedded_weights()
    network.save_weights(embedded_weights)
    network.save_labels(labels)
    
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1]):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = embedded_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(pt.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

def encode_dataframe_cv(df, target_name, batch_size, train_ratio,
                        epochs, network_layers, verbose, model_path):        
    X, y = preprocessor.get_X_y(df, target_name)
    X, labels = preprocessor.encode_vector_label(X)
    y = np.array(y)

    network = neural_embedder.NeuralEmbedder(df=df, target_name=target_name, epochs=epochs,
                                             batch_size=batch_size, network_layers=network_layers,
                                             verbose=verbose, model_path=model_path)
    
    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
    es_callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss',
                                                   mode='min',
                                                   patience=3,
                                                   verbose=0)
    
    weights = []
    for train_index, valid_index in skf.split(X, y):
        X_train, X_valid = X[train_index,:], X[valid_index,:]
        y_train, y_valid = y[train_index], y[valid_index]
        
        _ = network.fit(X_train, y_train, X_valid, y_valid,
                        callbacks=[es_callback])
        embedded_weights = network.get_embedded_weights()
        weights.append(embedded_weights)
        
    new_weights = list()
    for weights_list_tuple in zip(*weights):
        new_weights.append(
            [np.array(weights_).mean(axis=0)\
                for weights_ in zip(*weights_list_tuple)])
    
    network.save_weights(new_weights)
    network.save_labels(labels)
        
    if ENABLE_EMB_VIZ:
        network.make_visualizations_from_network(extension='png')

    df_to_enc = df.drop(target_name, axis=1)
    for index in range(df_to_enc.shape[1]):
        column = df_to_enc.columns[index]
        labels_column = labels[index]
        embeddings_column = new_weights[index]
        pca = PCA(n_components=1)
        Y = pca.fit_transform(embeddings_column)
        y_array = np.concatenate(Y)
        mapping = dict(zip(labels_column.classes_, y_array))
        file_writer.write_mapping(mapping,
                                Path.joinpath(pt.PROCESSED_DATA_DIR, 'embeddings'),
                                f'{target_name.lower()}_{column}.csv')
        df_to_enc[column] = df_to_enc[column].replace(to_replace=mapping)
        
    return df_to_enc

if __name__ == "__main__":
    main()