diff --git a/Makefile b/Makefile index 1c3842b..95cb8e6 100644 --- a/Makefile +++ b/Makefile @@ -1,65 +1,65 @@ run: - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_both_1 --epochs 2 --depth flat1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_both_1 --epochs 2 --depth flat1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type final --model_output both - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_both_2 --epochs 2 --depth flat1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_both_2 --epochs 2 --depth flat1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type inter --model_output both - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_both_3 --epochs 2 --depth deep1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_both_3 --epochs 2 --depth deep1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type final --model_output both - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_both_4 --epochs 2 --depth deep1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_both_4 --epochs 2 --depth deep1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type inter --model_output both - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_both_5 --epochs 2 --depth flat2 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_both_5 --epochs 2 --depth flat2 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type staggered --model_output both - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_client_1 --epochs 2 --depth flat2 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_client_1 --epochs 2 --depth flat2 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type final --model_output client - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_client_2 --epochs 2 --depth flat2 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_client_2 --epochs 2 --depth flat2 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --type inter --model_output client - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_client_3 --epochs 2 --depth deep1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_client_3 --epochs 2 --depth deep1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --type final --model_output client - python3 main.py --mode train --train data/rk_mini.csv.gz --model results/test/test_client_4 --epochs 2 --depth deep1 \ + python3 main.py --mode train --data data/rk_mini.csv.gz --model results/test/test_client_4 --epochs 2 --depth deep1 \ --filter_embd 32 --kernel_embd 3 --filter_main 16 --kernel_main 3 --dense_main 32 \ --dense_embd 16 --domain_embd 8 --batch 64 --balanced_weights --type inter --model_output client test: - python3 main.py --mode test --batch 128 --models results/test/test_both_* --test data/rk_mini.csv.gz --model_output both - python3 main.py --mode test --batch 128 --models results/test/test_client_* --test data/rk_mini.csv.gz --model_output client + python3 main.py --mode test --batch 128 --models results/test/test_both_* --data data/rk_mini.csv.gz --model_output both + python3 main.py --mode test --batch 128 --models results/test/test_client_* --data data/rk_mini.csv.gz --model_output client fancy: - python3 main.py --mode fancy --batch 128 --model results/test/test_both_1 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_both_1 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_both_2 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_both_2 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_both_3 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_both_3 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_both_4 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_both_4 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_both_5 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_both_5 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_client_1 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_client_1 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_client_2 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_client_2 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_client_3 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_client_3 --data data/rk_mini.csv.gz - python3 main.py --mode fancy --batch 128 --model results/test/test_client_4 --test data/rk_mini.csv.gz + python3 main.py --mode fancy --batch 128 --model results/test/test_client_4 --data data/rk_mini.csv.gz all-fancy: - python3 main.py --mode all_fancy --batch 128 --models results/test/test* --test data/rk_mini.csv.gz \ + python3 main.py --mode all_fancy --batch 128 --models results/test/test* --data data/rk_mini.csv.gz \ --out-prefix results/test/ hyper: diff --git a/hyperband.py b/hyperband.py index 835e844..04e774b 100644 --- a/hyperband.py +++ b/hyperband.py @@ -71,7 +71,7 @@ class Hyperband: shuffle=True, validation_split=0.4) - return {"loss": history.history['val_loss'][-1], + return {"loss": np.min(history.history['val_loss']), "early_stop": len(history.history["loss"]) < n_iterations} # can be called multiple times diff --git a/main.py b/main.py index 06b6b4d..06d038c 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,8 @@ import numpy as np import pandas as pd import tensorflow as tf from keras.callbacks import CSVLogger, EarlyStopping, ModelCheckpoint -from keras.models import Model, load_model as load_keras_model +from keras.models import Model +from sklearn.metrics import confusion_matrix import arguments import dataset @@ -14,7 +15,7 @@ import models # create logger import visualize from arguments import get_model_args -from utils import exists_or_make_path, get_custom_class_weights +from utils import exists_or_make_path, get_custom_class_weights, load_model logger = logging.getLogger('logger') logger.setLevel(logging.DEBUG) @@ -85,19 +86,13 @@ def create_model(model, output_type): raise Exception("unknown model output") -def load_model(path, custom_objects=None): - clf = load_keras_model(path, custom_objects) - embd = clf.get_layer("domain_cnn").layer - return embd, clf - - def main_paul_best(): pauls_best_params = models.pauls_networks.best_config main_train(pauls_best_params) def main_hyperband(): - params = { + param_dist = { # static params "type": [args.model_type], "depth": [args.model_depth], @@ -117,22 +112,25 @@ def main_hyperband(): "kernel_main": [1, 3, 5, 7, 9], "dense_main": [2 ** x for x in range(1, 12)], } - + logger.info("create training dataset") - domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.train_h5data, - args.train_data, + domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.data, + args.data, args.domain_length, args.window) server_tr = np.max(server_windows_tr, axis=1) - + if args.model_type in ("inter", "staggered"): server_tr = np.expand_dims(server_windows_tr, 2) - - hp = hyperband.Hyperband(params, + + hp = hyperband.Hyperband(param_dist, [domain_tr, flow_tr], - [client_tr, server_tr]) + [client_tr, server_tr], + max_iter=81, + savefile=args.hyperband_results) results = hp.run() - joblib.dump(results, args.hyperband_results) + + return results def main_train(param=None): @@ -140,8 +138,8 @@ def main_train(param=None): exists_or_make_path(args.model_path) logger.info(f"Use command line arguments: {args}") - domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.train_h5data, - args.train_data, + domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.data, + args.data, args.domain_length, args.window) logger.info("define callbacks") @@ -236,9 +234,9 @@ def main_retrain(): logger.info(f"Use command line arguments: {args}") exists_or_make_path(args.model_destination) - - domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.train_h5data, - args.train_data, + + domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.data, + args.data, args.domain_length, args.window) logger.info("define callbacks") @@ -265,7 +263,7 @@ def main_retrain(): custom_class_weights = None logger.info(f"Load pretrained model") - embedding, model = load_model(source, custom_objects=models.get_metrics()) + embedding, model = load_model(source, custom_objects=models.get_custom_objects()) if args.model_type in ("inter", "staggered"): server_tr = np.expand_dims(server_windows_tr, 2) @@ -293,17 +291,17 @@ def main_retrain(): def main_test(): logger.info("start test: load data") - domain_val, flow_val, _, _, _, _ = dataset.load_or_generate_raw_h5data(args.test_h5data, - args.test_data, + domain_val, flow_val, _, _, _, _ = dataset.load_or_generate_raw_h5data(args.data, + args.data, args.domain_length, args.window) - domain_encs, _ = dataset.load_or_generate_domains(args.test_data, args.domain_length) + domain_encs, _ = dataset.load_or_generate_domains(args.data, args.domain_length) for model_args in get_model_args(args): results = {} logger.info(f"process model {model_args['model_path']}") - embd_model, clf_model = load_model(model_args["clf_model"], custom_objects=models.get_metrics()) - + embd_model, clf_model = load_model(model_args["clf_model"], custom_objects=models.get_custom_objects()) + pred = clf_model.predict([domain_val, flow_val], batch_size=args.batch_size, verbose=1) @@ -324,8 +322,28 @@ def main_test(): def main_visualization(): - _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.test_h5data, - args.test_data, + def plot_model(clf_model, path): + embd, model = load_model(clf_model, custom_objects=models.get_custom_objects()) + visualize.plot_model_as(embd, os.path.join(path, "model_embd.pdf")) + visualize.plot_model_as(model, os.path.join(path, "model_clf.pdf")) + + def vis(model_name, model_path, df, df_paul, aggregation, curve): + visualize.plot_clf() + if aggregation == "user": + df = df.groupby(df.names).max() + df_paul = df_paul.groupby(df_paul.names).max() + if curve == "prc": + visualize.plot_precision_recall(df.client_val.as_matrix(), df.client_pred.as_matrix(), model_name) + visualize.plot_precision_recall(df_paul.client_val.as_matrix(), df_paul.client_pred.as_matrix(), "paul") + elif curve == "roc": + visualize.plot_roc_curve(df.client_val.as_matrix(), df.client_pred.as_matrix(), model_name) + visualize.plot_roc_curve(df_paul.client_val.as_matrix(), df_paul.client_pred.as_matrix(), "paul") + + visualize.plot_legend() + visualize.plot_save("{}/{}_{}.png".format(model_path, aggregation, curve)) + + _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.data, + args.data, args.domain_length, args.window) @@ -336,19 +354,17 @@ def main_visualization(): }) df["client_val"] = np.logical_or(df.hits_vt == 1.0, df.hits_trusted >= 3) df_user = df.groupby(df.names).max() - + paul = dataset.load_predictions("results/paul/") df_paul = pd.DataFrame(data={ "names": paul["testNames"].flatten(), "client_pred": paul["testScores"].flatten(), "hits_vt": paul["testLabel"].flatten(), "hits_trusted": paul["testHits"].flatten() }) df_paul["client_val"] = np.logical_or(df_paul.hits_vt == 1.0, df_paul.hits_trusted >= 3) - df_paul_user = df_paul.groupby(df_paul.names).max() logger.info("plot model") - embd, model = load_model(args.clf_model, custom_objects=models.get_metrics()) - visualize.plot_model_as(model, os.path.join(args.model_path, "model.png")) - + plot_model(args.clf_model, args.model_path) + # logger.info("plot training curve") # logs = pd.read_csv(args.train_log) # if "acc" in logs.keys(): @@ -359,31 +375,15 @@ def main_visualization(): # else: # logger.warning("Error while plotting training curves") - logger.info("plot pr curve") - visualize.plot_clf() - visualize.plot_precision_recall(df.client_val.as_matrix(), df.client_pred.as_matrix(), args.model_name) - visualize.plot_precision_recall(df_paul.client_val.as_matrix(), df_paul.client_pred.as_matrix(), "paul") - visualize.plot_legend() - visualize.plot_save("{}/window_client_prc.png".format(args.model_path)) + logger.info("plot window prc") + vis(args.model_name, args.model_path, df, df_paul, "window", "prc") + logger.info("plot window roc") + vis(args.model_name, args.model_path, df, df_paul, "window", "roc") + logger.info("plot user prc") + vis(args.model_name, args.model_path, df, df_paul, "user", "prc") + logger.info("plot user roc") + vis(args.model_name, args.model_path, df, df_paul, "user", "roc") - logger.info("plot roc curve") - visualize.plot_clf() - visualize.plot_roc_curve(df.client_val.as_matrix(), df.client_pred.as_matrix(), args.model_name) - visualize.plot_roc_curve(df_paul.client_val.as_matrix(), df_paul.client_pred.as_matrix(), "paul") - visualize.plot_legend() - visualize.plot_save("{}/window_client_roc.png".format(args.model_path)) - - visualize.plot_clf() - visualize.plot_precision_recall(df_user.client_val.as_matrix(), df_user.client_pred.as_matrix(), args.model_name) - visualize.plot_precision_recall(df_paul_user.client_val.as_matrix(), df_paul_user.client_pred.as_matrix(), "paul") - visualize.plot_legend() - visualize.plot_save("{}/user_client_prc.png".format(args.model_path)) - - visualize.plot_clf() - visualize.plot_roc_curve(df_user.client_val.as_matrix(), df_user.client_pred.as_matrix(), args.model_name) - visualize.plot_roc_curve(df_paul_user.client_val.as_matrix(), df_paul_user.client_pred.as_matrix(), "paul") - visualize.plot_legend() - visualize.plot_save("{}/user_client_roc.png".format(args.model_path)) # absolute values visualize.plot_confusion_matrix(df.client_val.as_matrix(), df.client_pred.as_matrix().round(), "{}/client_cov.png".format(args.model_path), @@ -398,25 +398,18 @@ def main_visualization(): visualize.plot_confusion_matrix(df_user.client_val.as_matrix(), df_user.client_pred.as_matrix().round(), "{}/user_cov_norm.png".format(args.model_path), normalize=True, title="User Confusion Matrix") - logger.info("visualize embedding") - domain_encs, labels = dataset.load_or_generate_domains(args.test_data, args.domain_length) - domain_embedding = results["domain_embds"] - visualize.plot_embedding(domain_embedding, labels, path="{}/embd_svd.png".format(args.model_path), method="svd") - visualize.plot_embedding(domain_embedding, labels, path="{}/embd_tsne.png".format(args.model_path), method="tsne") + plot_embedding(args.model_path, results["domain_embds"], args.data, args.domain_length) -def plot_embedding(): +def plot_embedding(model_path, domain_embedding, data, domain_length): logger.info("visualize embedding") - results = dataset.load_predictions(args.model_path) - domain_encs, labels = dataset.load_or_generate_domains(args.test_data, args.domain_length) - domain_embedding = results["domain_embds"] - visualize.plot_embedding(domain_embedding, labels, path="{}/embd_svd.png".format(args.model_path), method="svd") - visualize.plot_embedding(domain_embedding, labels, path="{}/embd_tsne.png".format(args.model_path), method="tsne") + domain_encs, labels = dataset.load_or_generate_domains(data, domain_length) + visualize.plot_embedding(domain_embedding, labels, path="{}/embd_svd.png".format(model_path), method="svd") def main_visualize_all(): - _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.test_h5data, - args.test_data, + _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.data, + args.data, args.domain_length, args.window) @@ -428,7 +421,7 @@ def main_visualize_all(): }) res["client_val"] = np.logical_or(res.hits_vt == 1.0, res.hits_trusted >= 3) return res - + paul = dataset.load_predictions("results/paul/") df_paul = pd.DataFrame(data={ "names": paul["testNames"].flatten(), "client_pred": paul["testScores"].flatten(), @@ -480,8 +473,8 @@ import joblib def main_beta(): - _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.test_h5data, - args.test_data, + _, _, name_val, hits_vt, hits_trusted, server_val = dataset.load_or_generate_raw_h5data(args.data, + args.data, args.domain_length, args.window) path, model_prefix = os.path.split(os.path.normpath(args.output_prefix)) @@ -489,7 +482,7 @@ def main_beta(): results = joblib.load(f"{path}/curves.joblib") except Exception: results = {} - results[model_prefix] = {} + results[model_prefix] = {"all": {}} def load_df(path): res = dataset.load_predictions(path) @@ -514,7 +507,9 @@ def main_beta(): for model_args in get_model_args(args): df = load_df(model_args["model_path"]) predictions.append(df.client_pred.as_matrix()) - results[model_prefix]["window_prc"] = visualize.calc_pr_mean(df.client_val.as_matrix(), predictions) + results[model_prefix][model_args["model_name"]] = confusion_matrix(df.client_val.as_matrix(), + df.client_pred.as_matrix().round()) + results[model_prefix]["all"]["window_prc"] = visualize.calc_pr_mean(df.client_val.as_matrix(), predictions) visualize.plot_pr_mean(df.client_val.as_matrix(), predictions, "mean") visualize.plot_pr_mean(df_paul.client_val.as_matrix(), [df_paul.client_pred.as_matrix()], "paul") visualize.plot_legend() @@ -526,7 +521,9 @@ def main_beta(): for model_args in get_model_args(args): df = load_df(model_args["model_path"]) predictions.append(df.client_pred.as_matrix()) - results[model_prefix]["window_roc"] = visualize.calc_roc_mean(df.client_val.as_matrix(), predictions) + results[model_prefix][model_args["model_name"]] = confusion_matrix(df.client_val.as_matrix(), + df.client_pred.as_matrix().round()) + results[model_prefix]["all"]["window_roc"] = visualize.calc_roc_mean(df.client_val.as_matrix(), predictions) visualize.plot_roc_mean(df.client_val.as_matrix(), predictions, "mean") visualize.plot_roc_mean(df_paul.client_val.as_matrix(), [df_paul.client_pred.as_matrix()], "paul") visualize.plot_legend() @@ -539,7 +536,9 @@ def main_beta(): df = load_df(model_args["model_path"]) df = df.groupby(df.names).max() predictions.append(df.client_pred.as_matrix()) - results[model_prefix]["user_prc"] = visualize.calc_pr_mean(df.client_val.as_matrix(), predictions) + results[model_prefix][model_args["model_name"]] = confusion_matrix(df.client_val.as_matrix(), + df.client_pred.as_matrix().round()) + results[model_prefix]["all"]["user_prc"] = visualize.calc_pr_mean(df.client_val.as_matrix(), predictions) visualize.plot_pr_mean(df.client_val.as_matrix(), predictions, "mean") visualize.plot_pr_mean(df_paul_user.client_val.as_matrix(), [df_paul_user.client_pred.as_matrix()], "paul") visualize.plot_legend() @@ -552,14 +551,14 @@ def main_beta(): df = load_df(model_args["model_path"]) df = df.groupby(df.names).max() predictions.append(df.client_pred.as_matrix()) - results[model_prefix]["user_roc"] = visualize.calc_roc_mean(df.client_val.as_matrix(), predictions) + results[model_prefix]["all"]["user_roc"] = visualize.calc_roc_mean(df.client_val.as_matrix(), predictions) visualize.plot_roc_mean(df.client_val.as_matrix(), predictions, "mean") visualize.plot_roc_mean(df_paul_user.client_val.as_matrix(), [df_paul_user.client_pred.as_matrix()], "paul") visualize.plot_legend() visualize.plot_save(f"{args.output_prefix}_user_client_roc_all.png") joblib.dump(results, f"{path}/curves.joblib") - + plot_overall_result() @@ -576,7 +575,7 @@ def plot_overall_result(): logger.info(f"plot {vis}") visualize.plot_clf() for model_key in results.keys(): - ys_mean, ys_std, score = results[model_key][vis] + ys_mean, ys_std, score = results[model_key]["all"][vis] plt.plot(x, ys_mean, label=f"{model_key} - {score:5.4}") plt.fill_between(x, ys_mean - ys_std, ys_mean + ys_std, alpha=0.2) if vis.endswith("prc"): @@ -604,14 +603,8 @@ def main(): main_visualization() if "all_fancy" == args.mode: main_visualize_all() - if "embd" == args.mode: - plot_embedding() - if "paul" == args.mode: - main_paul_best() if "beta" == args.mode: main_beta() - if "beta_all" == args.mode: - plot_overall_result() if __name__ == "__main__": diff --git a/utils.py b/utils.py index ccab16b..fc8d4c9 100644 --- a/utils.py +++ b/utils.py @@ -3,6 +3,7 @@ from operator import itemgetter import joblib import numpy as np +from keras.models import load_model as load_keras_model from sklearn.utils import class_weight @@ -27,3 +28,15 @@ def get_custom_sample_weights(client, server): def load_ordered_hyperband_results(path): results = joblib.load(path) return sorted(results, itemgetter("loss")) + + +def load_model(path, custom_objects=None): + clf = load_keras_model(path, custom_objects) + try: + embd = clf.get_layer("domain_cnn").layer + except Exception: + # in some version i forgot to specify domain_cnn + # this bug fix is for certain compatibility + embd = clf.layers[1].layer + + return embd, clf diff --git a/visualize.py b/visualize.py index e0e140b..39ad3a1 100644 --- a/visualize.py +++ b/visualize.py @@ -38,6 +38,7 @@ def plot_clf(): def plot_save(path, dpi=300): + plt.title(path) fig = plt.gcf() fig.set_size_inches(18.5, 10.5) fig.savefig(path, dpi=dpi) @@ -48,6 +49,10 @@ def plot_legend(): plt.legend() +def mathews_correlation_curve(y, y_pred): + pass + + def plot_precision_recall(y, y_pred, label=""): y = y.flatten() y_pred = y_pred.flatten()