rene/ma_cisco_malware
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add visualization for training curves, pr, roc

Browse Source
This commit is contained in:
René Knaebel 2017-07-14 14:58:17 +02:00
parent d0418b9efa
commit b35f23e518
7 changed files with 299 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -101,3 +101,6 @@ ENV/
*.csv
*.csv.gz
*.csv.tar.*
*.h5
*.npy
*.png

10
Makefile
View File

@ -1,5 +1,11 @@
run:
python3 main.py --modes train --batch 128 --model results/test --train data/rk_mini.csv.gz --epochs 10
test:
python3 main.py --modes train --epochs 1 --batch 128 --train data/rk_mini.csv.gz
python3 main.py --modes test --batch 128 --model results/test --test data/rk_mini.csv.gz
fancy:
python3 main.py --modes fancy --batch 128 --model results/test --test data/rk_mini.csv.gz
hyper:
python3 main.py --modes hyperband --epochs 1 --batch 64 --train data/rk_data.csv.gz
python3 main.py --modes hyperband --batch 64 --train data/rk_data.csv.gz

24
arguments.py
View File

@ -7,20 +7,14 @@ parser.add_argument("--modes", action="store", dest="modes", nargs="+",
default=[])
parser.add_argument("--train", action="store", dest="train_data",
default="data/full_dataset.csv.tar.bz2")
default="data/full_dataset.csv.tar.gz")
parser.add_argument("--test", action="store", dest="test_data",
default="data/full_future_dataset.csv.tar.bz2")
default="data/full_future_dataset.csv.tar.gz")
# parser.add_argument("--h5data", action="store", dest="h5data",
# default="")
#
parser.add_argument("--models", action="store", dest="model_path",
default="models/models_x")
parser.add_argument("--model", action="store", dest="model_path",
default="results/model_x")
# parser.add_argument("--pred", action="store", dest="pred",
# default="")
#
parser.add_argument("--type", action="store", dest="model_type",
default="paul")
@ -66,13 +60,17 @@ parser.add_argument("--domain_embd", action="store", dest="domain_embedding",
#
# parser.add_argument("--tmp", action="store_true", dest="tmp")
#
# parser.add_argument("--test", action="store_true", dest="test")
parser.add_argument("--stop_early", action="store_true", dest="stop_early")
parser.add_argument("--gpu", action="store_true", dest="gpu")
def parse():
args = parser.parse_args()
args.embedding_model = os.path.join(args.model_path, "embd.h5")
args.clf_model = os.path.join(args.model_path, "clf.h5")
args.train_log = os.path.join(args.model_path, "train.log")
args.h5data = args.train_data + ".h5"
args.train_log = os.path.join(args.model_path, "train.log.csv")
args.train_h5data = args.train_data + ".h5"
args.test_h5data = args.test_data + ".h5"
args.future_prediction = os.path.join(args.model_path, "future_predict.npy")
return args

18
dataset.py
View File

@ -199,3 +199,21 @@ def get_flow_per_user(df):
users = df['user_hash'].unique().tolist()
for user in users:
yield df.loc[df.user_hash == user]
def load_or_generate_h5data(h5data, train_data, domain_length, window_size):
char_dict = get_character_dict()
logger.info(f"check for h5data {h5data}")
try:
open(h5data, "r")
except FileNotFoundError:
logger.info("h5 data not found - load csv file")
user_flow_df = get_user_flow_data(train_data)
logger.info("create training dataset")
domain_tr, flow_tr, client_tr, server_tr = create_dataset_from_flows(user_flow_df, char_dict,
max_len=domain_length,
window_size=window_size)
logger.info("store training dataset as h5 file")
store_h5dataset(h5data, domain_tr, flow_tr, client_tr, server_tr)
logger.info("load h5 dataset")
return load_h5dataset(h5data)

132
main.py
View File

@ -3,6 +3,8 @@ import logging
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from keras.callbacks import ModelCheckpoint, CSVLogger, EarlyStopping
from keras.models import load_model
@ -10,8 +12,10 @@ import arguments
import dataset
import hyperband
import models
# create logger
import visualize
from dataset import load_or_generate_h5data
logger = logging.getLogger('logger')
logger.setLevel(logging.DEBUG)
@ -40,13 +44,15 @@ ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)
print = logger.info
args = arguments.parse()
# config = tf.ConfigProto(log_device_placement=True)
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
# config.gpu_options.allow_growth = True
# session = tf.Session(config=config)
if args.gpu:
config = tf.ConfigProto(log_device_placement=True)
config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
def exists_or_make_path(p):
@ -56,32 +62,13 @@ def exists_or_make_path(p):
def main_paul_best():
char_dict = dataset.get_character_dict()
domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data,
args.domain_length, args.window)
param = models.pauls_networks.best_config
param["vocab_size"] = len(char_dict) + 1
embedding, model = models.get_models_by_params(param)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit([domain_tr, flow_tr],
[client_tr, server_tr],
batch_size=args.batch_size,
epochs=args.epochs,
shuffle=True,
validation_split=0.2)
embedding.save(args.embedding_model)
model.save(args.clf_model)
pauls_best_params = models.pauls_networks.best_config
pauls_best_params["vocab_size"] = len(char_dict) + 1
main_train(pauls_best_params)
def main_hyperband():
char_dict = dataset.get_character_dict()
user_flow_df = dataset.get_user_flow_data(args.train_data)
params = {
# static params
@ -105,7 +92,7 @@ def main_hyperband():
}
logger.info("create training dataset")
domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data,
domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.train_h5data, args.train_data,
args.domain_length, args.window)
hp = hyperband.Hyperband(params,
[domain_tr, flow_tr],
@ -114,33 +101,15 @@ def main_hyperband():
json.dump(results, open("hyperband.json"))
def load_or_generate_h5data(h5data, train_data, domain_length, window_size):
char_dict = dataset.get_character_dict()
logger.info(f"check for h5data {h5data}")
try:
open(h5data, "r")
except FileNotFoundError:
logger.info("h5 data not found - load csv file")
user_flow_df = dataset.get_user_flow_data(train_data)
logger.info("create training dataset")
domain_tr, flow_tr, client_tr, server_tr = dataset.create_dataset_from_flows(user_flow_df, char_dict,
max_len=domain_length,
window_size=window_size)
logger.info("store training dataset as h5 file")
dataset.store_h5dataset(args.h5data, domain_tr, flow_tr, client_tr, server_tr)
logger.info("load h5 dataset")
return dataset.load_h5dataset(h5data)
def main_train():
def main_train(param=None):
exists_or_make_path(args.model_path)
char_dict = dataset.get_character_dict()
domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.h5data, args.train_data,
domain_tr, flow_tr, client_tr, server_tr = load_or_generate_h5data(args.train_h5data, args.train_data,
args.domain_length, args.window)
# parameter
param = {
p = {
"type": "paul",
"batch_size": 64,
"window_size": args.window,
@ -160,29 +129,34 @@ def main_train():
'kernels_main': 3,
'input_length': 40
}
if not param:
param = p
embedding, model = models.get_models_by_params(param)
embedding.summary()
model.summary()
logger.info("define callbacks")
cp = ModelCheckpoint(filepath=args.clf_model,
callbacks = []
callbacks.append(ModelCheckpoint(filepath=args.clf_model,
monitor='val_loss',
verbose=False,
save_best_only=True)
csv = CSVLogger(args.train_log)
early = EarlyStopping(monitor='val_loss',
save_best_only=True))
callbacks.append(CSVLogger(args.train_log))
if args.stop_early:
callbacks.append(EarlyStopping(monitor='val_loss',
patience=5,
verbose=False)
verbose=False))
logger.info("compile model")
custom_metrics = models.get_metric_functions()
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
metrics=['accuracy'] + custom_metrics)
logger.info("start training")
model.fit([domain_tr, flow_tr],
[client_tr, server_tr],
batch_size=args.batch_size,
epochs=args.epochs,
callbacks=[cp, csv, early],
callbacks=callbacks,
shuffle=True,
validation_split=0.2)
logger.info("save embedding")
@ -190,42 +164,46 @@ def main_train():
def main_test():
domain_val, flow_val, client_val, server_val = load_or_generate_h5data(args.h5data, args.train_data,
domain_val, flow_val, client_val, server_val = load_or_generate_h5data(args.test_h5data, args.test_data,
args.domain_length, args.window)
clf = load_model(args.clf_model)
loss, _, _, client_acc, server_acc = clf.evaluate([domain_val, flow_val],
clf = load_model(args.clf_model, custom_objects=models.get_metrics())
stats = clf.evaluate([domain_val, flow_val],
[client_val, server_val],
batch_size=args.batch_size)
logger.info(f"loss: {loss}\nclient acc: {client_acc}\nserver acc: {server_acc}")
# logger.info(f"loss: {loss}\nclient acc: {client_acc}\nserver acc: {server_acc}")
logger.info(stats)
y_pred = clf.predict([domain_val, flow_val],
batch_size=args.batch_size)
np.save(os.path.join(args.model_path, "future_predict.npy"), y_pred)
np.save(args.future_prediction, y_pred)
def main_visualization():
mask = dataset.load_mask_eval(args.data, args.test_image)
y_pred_path = args.model_path + "pred.npy"
_, _, client_val, server_val = load_or_generate_h5data(args.test_h5data, args.test_data,
args.domain_length, args.window)
logger.info("plot model")
model = load_model(args.model_path + "model.h5",
custom_objects=evaluation.get_metrics())
model = load_model(args.clf_model, custom_objects=models.get_metrics())
visualize.plot_model(model, args.model_path + "model.png")
logger.info("plot training curve")
logs = pd.read_csv(args.model_path + "train.log")
visualize.plot_training_curve(logs, "{}/train.png".format(args.model_path))
pred = np.load(y_pred_path)
logs = pd.read_csv(args.train_log)
visualize.plot_training_curve(logs, "client", "{}/client_train.png".format(args.model_path))
visualize.plot_training_curve(logs, "server", "{}/server_train.png".format(args.model_path))
client_pred, server_pred = np.load(args.future_prediction)
logger.info("plot pr curve")
visualize.plot_precision_recall(mask, pred, "{}/prc.png".format(args.model_path))
visualize.plot_precision_recall_curves(mask, pred, "{}/prc2.png".format(args.model_path))
visualize.plot_precision_recall(client_val.value, client_pred, "{}/client_prc.png".format(args.model_path))
visualize.plot_precision_recall(server_val.value, server_pred, "{}/server_prc.png".format(args.model_path))
visualize.plot_precision_recall_curves(client_val.value, client_pred, "{}/client_prc2.png".format(args.model_path))
visualize.plot_precision_recall_curves(server_val.value, server_pred, "{}/server_prc2.png".format(args.model_path))
logger.info("plot roc curve")
visualize.plot_roc_curve(mask, pred, "{}/roc.png".format(args.model_path))
logger.info("store prediction image")
visualize.save_image_as(pred, "{}/pred.png".format(args.model_path))
visualize.plot_roc_curve(client_val.value, client_pred, "{}/client_roc.png".format(args.model_path))
visualize.plot_roc_curve(server_val.value, server_pred, "{}/server_roc.png".format(args.model_path))
def main_score():
mask = dataset.load_mask_eval(args.data, args.test_image)
pred = np.load(args.pred)
visualize.score_model(mask, pred)
# mask = dataset.load_mask_eval(args.data, args.test_image)
# pred = np.load(args.pred)
# visualize.score_model(mask, pred)
pass
def main():

53
models/__init__.py
View File

@ -1,3 +1,6 @@
import keras.backend as K
import dataset
from . import pauls_networks
from . import renes_networks
@ -6,7 +9,7 @@ def get_models_by_params(params: dict):
# decomposing param section
# mainly embedding model
network_type = params.get("type")
vocab_size = params.get("vocab_size")
vocab_size = len(dataset.get_character_dict()) + 1
embedding_size = params.get("embedding_size")
input_length = params.get("input_length")
filter_embedding = params.get("filter_embedding")
@ -30,3 +33,51 @@ def get_models_by_params(params: dict):
filter_main, kernel_main, dense_dim, embedding_model)
return embedding_model, predict_model
def get_metrics():
return dict([
("precision", precision),
("recall", recall),
("f1_score", f1_score),
])
def get_metric_functions():
return [precision, recall, f1_score]
def precision(y_true, y_pred):
# Count positive samples.
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
return true_positives / (predicted_positives + K.epsilon())
def recall(y_true, y_pred):
# Count positive samples.
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
return true_positives / (possible_positives + K.epsilon())
def f1_score(y_true, y_pred):
return f_score(1)(y_true, y_pred)
def f05_score(y_true, y_pred):
return f_score(0.5)(y_true, y_pred)
def f_score(beta):
def _f(y_true, y_pred):
p = precision(y_true, y_pred)
r = recall(y_true, y_pred)
bb = beta ** 2
fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
return fbeta_score
return _f

146
visualize.py Normal file
View File

@ -0,0 +1,146 @@
import os
import matplotlib.pyplot as plt
import numpy as np
from keras.utils import plot_model
from sklearn.metrics import (
auc, classification_report, confusion_matrix, fbeta_score, precision_recall_curve,
roc_auc_score, roc_curve
)
def scores(y_true, y_pred):
for (path, dirnames, fnames) in os.walk("results/"):
for f in fnames:
if path[-1] == "1" and f.endswith("npy"):
y_pred = np.load(os.path.join(path, f)).flatten()
print(path)
tp = np.sum(np.logical_and(y_pred >= 0.5, y_true == 1))
tn = np.sum(np.logical_and(y_pred < 0.5, y_true == 0))
fp = np.sum(np.logical_and(y_pred >= 0.5, y_true == 0))
fn = np.sum(np.logical_and(y_pred < 0.5, y_true == 1))
precision = tp / (tp + fp)
recall = tp / (tp + fn)
accuracy = (tp + tn) / len(y_true)
f1_score = 2 * (precision * recall) / (precision + recall)
f05_score = (1 + 0.5 ** 2) * (precision * recall) / (0.5 ** 2 * precision + recall)
print(" precision:", precision)
print(" recall:", recall)
print(" accuracy:", accuracy)
print(" f1 score:", f1_score)
print(" f0.5 score:", f05_score)
def plot_precision_recall(mask, prediction, path):
y = mask.flatten()
y_pred = prediction.flatten()
precision, recall, thresholds = precision_recall_curve(y, y_pred)
decreasing_max_precision = np.maximum.accumulate(precision)[::-1]
plt.clf()
# fig, ax = plt.subplots(1, 1)
# ax.hold(True)
plt.plot(recall, precision, '--b')
# ax.step(recall[::-1], decreasing_max_precision, '-r')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.savefig(path, dpi=600)
plt.close()
def plot_precision_recall_curves(mask, prediction, path):
y = mask.flatten()
y_pred = prediction.flatten()
precision, recall, thresholds = precision_recall_curve(y, y_pred)
plt.clf()
plt.plot(recall, label="Recall")
plt.plot(precision, label="Precision")
plt.xlabel('Threshold')
plt.ylabel('Score')
plt.savefig(path, dpi=600)
plt.close()
def score_model(y, prediction):
y = y.flatten()
y_pred = prediction.flatten()
precision, recall, thresholds = precision_recall_curve(y, y_pred)
print(classification_report(y, y_pred.round()))
print("Area under PR curve", auc(recall, precision))
print("roc auc score", roc_auc_score(y, y_pred))
print("F1 Score", fbeta_score(y, y_pred.round(), 1))
print("F0.5 Score", fbeta_score(y, y_pred.round(), 0.5))
def plot_roc_curve(mask, prediction, path):
y = mask.flatten()
y_pred = prediction.flatten()
fpr, tpr, thresholds = roc_curve(y, y_pred)
roc_auc = auc(fpr, tpr)
plt.clf()
plt.plot(fpr, tpr)
plt.savefig(path, dpi=600)
plt.close()
print("roc_auc", roc_auc)
def plot_confusion_matrix(y_true, y_pred,
normalize=False,
title='Confusion matrix',
cmap="Blues"):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
plt.clf()
cm = confusion_matrix(y_true, y_pred)
classes = [0, 1]
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
thresh = cm.max() / 2.
for i, j in ((i, j) for i in range(cm.shape[0]) for j in range(cm.shape[1])):
plt.text(j, i, cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
def plot_training_curve(logs, key, path, dpi=600):
plt.clf()
plt.plot(logs[f"{key}_acc"], label="accuracy")
plt.plot(logs[f"{key}_f1_score"], label="f1_score")
plt.plot(logs[f"val_{key}_acc"], label="accuracy")
plt.plot(logs[f"val_{key}_f1_score"], label="val_f1_score")
plt.xlabel('epoch')
plt.ylabel('percentage')
plt.legend()
plt.savefig(path, dpi=dpi)
plt.close()
def plot_model_as(model, path):
plot_model(model, to_file=path, show_shapes=True, show_layer_names=True)