From b35f23e518ace43e93224f5a71d7088e37740197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Knaebel?= Date: Fri, 14 Jul 2017 14:58:17 +0200 Subject: [PATCH] add visualization for training curves, pr, roc --- .gitignore | 5 +- Makefile | 10 +++- arguments.py | 24 ++++---- dataset.py | 18 ++++++ main.py | 142 +++++++++++++++++++------------------------ models/__init__.py | 53 +++++++++++++++- visualize.py | 146 +++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 299 insertions(+), 99 deletions(-) create mode 100644 visualize.py diff --git a/.gitignore b/.gitignore index e834a02..2f6af39 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,7 @@ ENV/ *.joblib *.csv *.csv.gz -*.csv.tar.* \ No newline at end of file +*.csv.tar.* +*.h5 +*.npy +*.png diff --git a/Makefile b/Makefile index 4a2d47e..1a23113 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,11 @@ +run: + python3 main.py --modes train --batch 128 --model results/test --train data/rk_mini.csv.gz --epochs 10 + test: - python3 main.py --modes train --epochs 1 --batch 128 --train data/rk_mini.csv.gz + python3 main.py --modes test --batch 128 --model results/test --test data/rk_mini.csv.gz + +fancy: + python3 main.py --modes fancy --batch 128 --model results/test --test data/rk_mini.csv.gz hyper: - python3 main.py --modes hyperband --epochs 1 --batch 64 --train data/rk_data.csv.gz + python3 main.py --modes hyperband --batch 64 --train data/rk_data.csv.gz diff --git a/arguments.py b/arguments.py index 740a778..d392d22 100644 --- a/arguments.py +++ b/arguments.py @@ -7,20 +7,14 @@ parser.add_argument("--modes", action="store", dest="modes", nargs="+", default=[]) parser.add_argument("--train", action="store", dest="train_data", - default="data/full_dataset.csv.tar.bz2") + default="data/full_dataset.csv.tar.gz") parser.add_argument("--test", action="store", dest="test_data", - default="data/full_future_dataset.csv.tar.bz2") + default="data/full_future_dataset.csv.tar.gz") -# parser.add_argument("--h5data", action="store", dest="h5data", -# default="") -# -parser.add_argument("--models", action="store", dest="model_path", - default="models/models_x") +parser.add_argument("--model", action="store", dest="model_path", + default="results/model_x") -# parser.add_argument("--pred", action="store", dest="pred", -# default="") -# parser.add_argument("--type", action="store", dest="model_type", default="paul") @@ -66,13 +60,17 @@ parser.add_argument("--domain_embd", action="store", dest="domain_embedding", # # parser.add_argument("--tmp", action="store_true", dest="tmp") # -# parser.add_argument("--test", action="store_true", dest="test") +parser.add_argument("--stop_early", action="store_true", dest="stop_early") +parser.add_argument("--gpu", action="store_true", dest="gpu") + def parse(): args = parser.parse_args() args.embedding_model = os.path.join(args.model_path, "embd.h5") args.clf_model = os.path.join(args.model_path, "clf.h5") - args.train_log = os.path.join(args.model_path, "train.log") - args.h5data = args.train_data + ".h5" + args.train_log = os.path.join(args.model_path, "train.log.csv") + args.train_h5data = args.train_data + ".h5" + args.test_h5data = args.test_data + ".h5" + args.future_prediction = os.path.join(args.model_path, "future_predict.npy") return args diff --git a/dataset.py b/dataset.py index c759b64..f00a360 100644 --- a/dataset.py +++ b/dataset.py @@ -199,3 +199,21 @@ def get_flow_per_user(df): users = df['user_hash'].unique().tolist() for user in users: yield df.loc[df.user_hash == user] + + +def load_or_generate_h5data(h5data, train_data, domain_length, window_size): + char_dict = get_character_dict() + logger.info(f"check for h5data {h5data}") + try: + open(h5data, "r") + except FileNotFoundError: + logger.info("h5 data not found - load csv file") + user_flow_df = get_user_flow_data(train_data) + logger.info("create training dataset") + domain_tr, flow_tr, client_tr, server_tr = create_dataset_from_flows(user_flow_df, char_dict, + max_len=domain_length, + window_size=window_size) + logger.info("store training dataset as h5 file") + store_h5dataset(h5data, domain_tr, flow_tr, client_tr, server_tr) + logger.info("load h5 dataset") + return load_h5dataset(h5data) diff --git a/main.py b/main.py index e7f8c98..55f1a5c 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,8 @@ import logging import os import numpy as np +import pandas as pd +import tensorflow as tf from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping from keras.models import load_model @@ -10,8 +12,10 @@ import arguments import dataset import hyperband import models - # create logger +import visualize +from dataset import load_or_generate_h5data + logger = logging.getLogger('logger') logger.setLevel(logging.DEBUG) @@ -40,13 +44,15 @@ ch.setFormatter(formatter) # add ch to logger logger.addHandler(ch) +print = logger.info + args = arguments.parse() - -# config = tf.ConfigProto(log_device_placement=True) -# config.gpu_options.per_process_gpu_memory_fraction = 0.5 -# config.gpu_options.allow_growth = True -# session = tf.Session(config=config) +if args.gpu: + config = tf.ConfigProto(log_device_placement=True) + config.gpu_options.per_process_gpu_memory_fraction = 0.5 + config.gpu_options.allow_growth = True + session = tf.Session(config=config) def exists_or_make_path(p): @@ -56,32 +62,13 @@ def exists_or_make_path(p): def main_paul_best(): char_dict = dataset.get_character_dict() - domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data, - args.domain_length, args.window) - - param = models.pauls_networks.best_config - param["vocab_size"] = len(char_dict) + 1 - - embedding, model = models.get_models_by_params(param) - - model.compile(optimizer='adam', - loss='categorical_crossentropy', - metrics=['accuracy']) - - model.fit([domain_tr, flow_tr], - [client_tr, server_tr], - batch_size=args.batch_size, - epochs=args.epochs, - shuffle=True, - validation_split=0.2) - - embedding.save(args.embedding_model) - model.save(args.clf_model) + pauls_best_params = models.pauls_networks.best_config + pauls_best_params["vocab_size"] = len(char_dict) + 1 + main_train(pauls_best_params) def main_hyperband(): char_dict = dataset.get_character_dict() - user_flow_df = dataset.get_user_flow_data(args.train_data) params = { # static params @@ -105,7 +92,7 @@ def main_hyperband(): } logger.info("create training dataset") - domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data, + domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.train_h5data, args.train_data, args.domain_length, args.window) hp = hyperband.Hyperband(params, [domain_tr, flow_tr], @@ -114,33 +101,15 @@ def main_hyperband(): json.dump(results, open("hyperband.json")) -def load_or_generate_h5data(h5data, train_data, domain_length, window_size): - char_dict = dataset.get_character_dict() - logger.info(f"check for h5data {h5data}") - try: - open(h5data, "r") - except FileNotFoundError: - logger.info("h5 data not found - load csv file") - user_flow_df = dataset.get_user_flow_data(train_data) - logger.info("create training dataset") - domain_tr, flow_tr, client_tr, server_tr = dataset.create_dataset_from_flows(user_flow_df, char_dict, - max_len=domain_length, - window_size=window_size) - logger.info("store training dataset as h5 file") - dataset.store_h5dataset(args.h5data, domain_tr, flow_tr, client_tr, server_tr) - logger.info("load h5 dataset") - return dataset.load_h5dataset(h5data) - - -def main_train(): +def main_train(param=None): exists_or_make_path(args.model_path) char_dict = dataset.get_character_dict() - domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data, + domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.train_h5data, args.train_data, args.domain_length, args.window) # parameter - param = { + p = { "type": "paul", "batch_size": 64, "window_size": args.window, @@ -160,29 +129,34 @@ def main_train(): 'kernels_main': 3, 'input_length': 40 } + if not param: + param = p embedding, model = models.get_models_by_params(param) embedding.summary() model.summary() logger.info("define callbacks") - cp = ModelCheckpoint(filepath=args.clf_model, - monitor='val_loss', - verbose=False, - save_best_only=True) - csv = CSVLogger(args.train_log) - early = EarlyStopping(monitor='val_loss', - patience=5, - verbose=False) + callbacks = [] + callbacks.append(ModelCheckpoint(filepath=args.clf_model, + monitor='val_loss', + verbose=False, + save_best_only=True)) + callbacks.append(CSVLogger(args.train_log)) + if args.stop_early: + callbacks.append(EarlyStopping(monitor='val_loss', + patience=5, + verbose=False)) logger.info("compile model") + custom_metrics = models.get_metric_functions() model.compile(optimizer='adam', loss='categorical_crossentropy', - metrics=['accuracy']) + metrics=['accuracy'] + custom_metrics) logger.info("start training") model.fit([domain_tr, flow_tr], [client_tr, server_tr], batch_size=args.batch_size, epochs=args.epochs, - callbacks=[cp, csv, early], + callbacks=callbacks, shuffle=True, validation_split=0.2) logger.info("save embedding") @@ -190,42 +164,46 @@ def main_train(): def main_test(): - domain_val, flow_val, client_val, server_val = load_or_generate_h5data(args.h5data, args.train_data, + domain_val, flow_val, client_val, server_val = load_or_generate_h5data(args.test_h5data, args.test_data, args.domain_length, args.window) - clf = load_model(args.clf_model) - loss, _, _, client_acc, server_acc = clf.evaluate([domain_val, flow_val], - [client_val, server_val], - batch_size=args.batch_size) - logger.info(f"loss: {loss}\nclient acc: {client_acc}\nserver acc: {server_acc}") + clf = load_model(args.clf_model, custom_objects=models.get_metrics()) + stats = clf.evaluate([domain_val, flow_val], + [client_val, server_val], + batch_size=args.batch_size) + # logger.info(f"loss: {loss}\nclient acc: {client_acc}\nserver acc: {server_acc}") + logger.info(stats) y_pred = clf.predict([domain_val, flow_val], batch_size=args.batch_size) - np.save(os.path.join(args.model_path, "future_predict.npy"), y_pred) + np.save(args.future_prediction, y_pred) def main_visualization(): - mask = dataset.load_mask_eval(args.data, args.test_image) - y_pred_path = args.model_path + "pred.npy" + _, _, client_val, server_val = load_or_generate_h5data(args.test_h5data, args.test_data, + args.domain_length, args.window) logger.info("plot model") - model = load_model(args.model_path + "model.h5", - custom_objects=evaluation.get_metrics()) + model = load_model(args.clf_model, custom_objects=models.get_metrics()) visualize.plot_model(model, args.model_path + "model.png") logger.info("plot training curve") - logs = pd.read_csv(args.model_path + "train.log") - visualize.plot_training_curve(logs, "{}/train.png".format(args.model_path)) - pred = np.load(y_pred_path) + logs = pd.read_csv(args.train_log) + visualize.plot_training_curve(logs, "client", "{}/client_train.png".format(args.model_path)) + visualize.plot_training_curve(logs, "server", "{}/server_train.png".format(args.model_path)) + + client_pred, server_pred = np.load(args.future_prediction) logger.info("plot pr curve") - visualize.plot_precision_recall(mask, pred, "{}/prc.png".format(args.model_path)) - visualize.plot_precision_recall_curves(mask, pred, "{}/prc2.png".format(args.model_path)) + visualize.plot_precision_recall(client_val.value, client_pred, "{}/client_prc.png".format(args.model_path)) + visualize.plot_precision_recall(server_val.value, server_pred, "{}/server_prc.png".format(args.model_path)) + visualize.plot_precision_recall_curves(client_val.value, client_pred, "{}/client_prc2.png".format(args.model_path)) + visualize.plot_precision_recall_curves(server_val.value, server_pred, "{}/server_prc2.png".format(args.model_path)) logger.info("plot roc curve") - visualize.plot_roc_curve(mask, pred, "{}/roc.png".format(args.model_path)) - logger.info("store prediction image") - visualize.save_image_as(pred, "{}/pred.png".format(args.model_path)) + visualize.plot_roc_curve(client_val.value, client_pred, "{}/client_roc.png".format(args.model_path)) + visualize.plot_roc_curve(server_val.value, server_pred, "{}/server_roc.png".format(args.model_path)) def main_score(): - mask = dataset.load_mask_eval(args.data, args.test_image) - pred = np.load(args.pred) - visualize.score_model(mask, pred) + # mask = dataset.load_mask_eval(args.data, args.test_image) + # pred = np.load(args.pred) + # visualize.score_model(mask, pred) + pass def main(): diff --git a/models/__init__.py b/models/__init__.py index fee9e4c..84228b9 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,3 +1,6 @@ +import keras.backend as K + +import dataset from . import pauls_networks from . import renes_networks @@ -6,7 +9,7 @@ def get_models_by_params(params: dict): # decomposing param section # mainly embedding model network_type = params.get("type") - vocab_size = params.get("vocab_size") + vocab_size = len(dataset.get_character_dict()) + 1 embedding_size = params.get("embedding_size") input_length = params.get("input_length") filter_embedding = params.get("filter_embedding") @@ -30,3 +33,51 @@ def get_models_by_params(params: dict): filter_main, kernel_main, dense_dim, embedding_model) return embedding_model, predict_model + + +def get_metrics(): + return dict([ + ("precision", precision), + ("recall", recall), + ("f1_score", f1_score), + ]) + + +def get_metric_functions(): + return [precision, recall, f1_score] + + +def precision(y_true, y_pred): + # Count positive samples. + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1))) + return true_positives / (predicted_positives + K.epsilon()) + + +def recall(y_true, y_pred): + # Count positive samples. + true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) + possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) + return true_positives / (possible_positives + K.epsilon()) + + +def f1_score(y_true, y_pred): + return f_score(1)(y_true, y_pred) + + +def f05_score(y_true, y_pred): + return f_score(0.5)(y_true, y_pred) + + +def f_score(beta): + def _f(y_true, y_pred): + p = precision(y_true, y_pred) + r = recall(y_true, y_pred) + + bb = beta ** 2 + + fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon()) + + return fbeta_score + + return _f diff --git a/visualize.py b/visualize.py new file mode 100644 index 0000000..229e35e --- /dev/null +++ b/visualize.py @@ -0,0 +1,146 @@ +import os + +import matplotlib.pyplot as plt +import numpy as np +from keras.utils import plot_model +from sklearn.metrics import ( + auc, classification_report, confusion_matrix, fbeta_score, precision_recall_curve, + roc_auc_score, roc_curve +) + + +def scores(y_true, y_pred): + for (path, dirnames, fnames) in os.walk("results/"): + for f in fnames: + if path[-1] == "1" and f.endswith("npy"): + y_pred = np.load(os.path.join(path, f)).flatten() + print(path) + tp = np.sum(np.logical_and(y_pred >= 0.5, y_true == 1)) + tn = np.sum(np.logical_and(y_pred < 0.5, y_true == 0)) + fp = np.sum(np.logical_and(y_pred >= 0.5, y_true == 0)) + fn = np.sum(np.logical_and(y_pred < 0.5, y_true == 1)) + precision = tp / (tp + fp) + recall = tp / (tp + fn) + accuracy = (tp + tn) / len(y_true) + f1_score = 2 * (precision * recall) / (precision + recall) + f05_score = (1 + 0.5 ** 2) * (precision * recall) / (0.5 ** 2 * precision + recall) + print(" precision:", precision) + print(" recall:", recall) + print(" accuracy:", accuracy) + print(" f1 score:", f1_score) + print(" f0.5 score:", f05_score) + + +def plot_precision_recall(mask, prediction, path): + y = mask.flatten() + y_pred = prediction.flatten() + precision, recall, thresholds = precision_recall_curve(y, y_pred) + decreasing_max_precision = np.maximum.accumulate(precision)[::-1] + + plt.clf() + # fig, ax = plt.subplots(1, 1) + # ax.hold(True) + plt.plot(recall, precision, '--b') + # ax.step(recall[::-1], decreasing_max_precision, '-r') + plt.xlabel('Recall') + plt.ylabel('Precision') + + plt.savefig(path, dpi=600) + plt.close() + + +def plot_precision_recall_curves(mask, prediction, path): + y = mask.flatten() + y_pred = prediction.flatten() + precision, recall, thresholds = precision_recall_curve(y, y_pred) + + plt.clf() + plt.plot(recall, label="Recall") + plt.plot(precision, label="Precision") + plt.xlabel('Threshold') + plt.ylabel('Score') + + plt.savefig(path, dpi=600) + plt.close() + + +def score_model(y, prediction): + y = y.flatten() + y_pred = prediction.flatten() + + precision, recall, thresholds = precision_recall_curve(y, y_pred) + + print(classification_report(y, y_pred.round())) + print("Area under PR curve", auc(recall, precision)) + print("roc auc score", roc_auc_score(y, y_pred)) + print("F1 Score", fbeta_score(y, y_pred.round(), 1)) + print("F0.5 Score", fbeta_score(y, y_pred.round(), 0.5)) + + +def plot_roc_curve(mask, prediction, path): + y = mask.flatten() + y_pred = prediction.flatten() + fpr, tpr, thresholds = roc_curve(y, y_pred) + roc_auc = auc(fpr, tpr) + plt.clf() + plt.plot(fpr, tpr) + plt.savefig(path, dpi=600) + plt.close() + + print("roc_auc", roc_auc) + + +def plot_confusion_matrix(y_true, y_pred, + normalize=False, + title='Confusion matrix', + cmap="Blues"): + """ + This function prints and plots the confusion matrix. + Normalization can be applied by setting `normalize=True`. + """ + plt.clf() + cm = confusion_matrix(y_true, y_pred) + classes = [0, 1] + plt.imshow(cm, interpolation='nearest', cmap=cmap) + plt.title(title) + plt.colorbar() + tick_marks = np.arange(len(classes)) + plt.xticks(tick_marks, classes, rotation=45) + plt.yticks(tick_marks, classes) + + if normalize: + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + print("Normalized confusion matrix") + else: + print('Confusion matrix, without normalization') + + print(cm) + + thresh = cm.max() / 2. + for i, j in ((i, j) for i in range(cm.shape[0]) for j in range(cm.shape[1])): + plt.text(j, i, cm[i, j], + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black") + + plt.tight_layout() + plt.ylabel('True label') + plt.xlabel('Predicted label') + + +def plot_training_curve(logs, key, path, dpi=600): + plt.clf() + plt.plot(logs[f"{key}_acc"], label="accuracy") + plt.plot(logs[f"{key}_f1_score"], label="f1_score") + + plt.plot(logs[f"val_{key}_acc"], label="accuracy") + plt.plot(logs[f"val_{key}_f1_score"], label="val_f1_score") + + plt.xlabel('epoch') + plt.ylabel('percentage') + plt.legend() + plt.savefig(path, dpi=dpi) + plt.close() + + +def plot_model_as(model, path): + plot_model(model, to_file=path, show_shapes=True, show_layer_names=True)