2017-06-30 10:12:20 +02:00
|
|
|
#!/usr/bin/python2
|
|
|
|
|
|
|
|
|
|
import joblib
|
2017-07-08 17:46:07 +02:00
|
|
|
import numpy as np
|
2017-06-30 10:49:49 +02:00
|
|
|
import pandas as pd
|
2017-06-30 10:12:20 +02:00
|
|
|
|
2017-06-30 18:43:50 +02:00
|
|
|
df = joblib.load("/mnt/projekte/pmlcluster/cisco/trainData/multipleTaskLearning/currentData.joblib")
|
2017-07-05 17:37:08 +02:00
|
|
|
df = pd.concat(df["data"])
|
2017-06-30 10:49:49 +02:00
|
|
|
df.reset_index(inplace=True)
|
2017-07-08 17:46:07 +02:00
|
|
|
df.dropna(axis=0, how="any", inplace=True)
|
|
|
|
|
df[["duration", "bytes_down", "bytes_up"]] = df[["duration", "bytes_down", "bytes_up"]].astype(np.int)
|
|
|
|
|
df[["domain", "server_ip"]] = df[["domain", "server_ip"]].astype(str)
|
|
|
|
|
df.serverLabel = df.serverLabel.astype(np.bool)
|
2017-07-11 11:12:03 +02:00
|
|
|
df.virusTotalHits = df.virusTotalHits.astype(np.int8)
|
|
|
|
|
df.trustedHits = df.trustedHits.astype(np.int8)
|
2017-07-08 17:46:07 +02:00
|
|
|
|
2017-07-05 17:37:08 +02:00
|
|
|
df.to_csv("/tmp/rk/full_future_dataset.csv.gz", compression="gzip")
|
