new dataset format: multi-lists -> two arrays

This commit is contained in:
René Knaebel 2017-07-04 09:18:50 +02:00
parent 5f8a760a0c
commit 5743127b7f
2 changed files with 65 additions and 44 deletions

View File

@ -5,10 +5,22 @@ import numpy as np
import pandas as pd import pandas as pd
from tqdm import tqdm from tqdm import tqdm
chars = dict((char, idx + 1) for (idx, char) in
enumerate(string.ascii_lowercase + string.punctuation + string.digits))
def get_character_dict(): def get_character_dict():
return dict((char, idx) for (idx, char) in return chars
enumerate(string.ascii_lowercase + string.punctuation))
def encode_char(c):
if c in chars:
return chars[c]
else:
return 0
encode_char = np.vectorize(encode_char)
def get_user_chunks(dataFrame, windowSize=10, overlapping=False, def get_user_chunks(dataFrame, windowSize=10, overlapping=False,
@ -49,16 +61,19 @@ def get_user_chunks(dataFrame, windowSize=10, overlapping=False,
curDomains = useData['domain'] curDomains = useData['domain']
outDomainLists.append(list(curDomains)) outDomainLists.append(list(curDomains))
outDFFrames.append(useData) outDFFrames.append(useData)
if len(outDomainLists[-1]) != windowSize:
outDomainLists.pop(-1)
outDFFrames.pop(-1)
return (outDomainLists, outDFFrames) return (outDomainLists, outDFFrames)
def get_domain_features(domain, vocab, max_length=40): def get_domain_features(domain, vocab, max_length=40):
curFeature = np.zeros([max_length, ]) encoding = np.zeros((max_length,))
for j in range(np.min([len(domain), max_length])): for j in range(np.min([len(domain), max_length])):
curCharacter = domain[-j] curCharacter = domain[-j]
if curCharacter in vocab: if curCharacter in vocab:
curFeature[j] = vocab[curCharacter] encoding[j] = vocab[curCharacter]
return curFeature return encoding
def get_flow_features(flow): def get_flow_features(flow):
@ -86,66 +101,76 @@ def get_cisco_features(curDataLine, urlSIPDict):
return np.zeros([numCiscoFeatures, ]).ravel() return np.zeros([numCiscoFeatures, ]).ravel()
def create_dataset_from_flows(user_flow_df, char_dict, maxLen, windowSize=10, use_cisco_features=False): def create_dataset_from_flows(user_flow_df, char_dict, max_len, window_size=10, use_cisco_features=False):
domainLists = [] domains = []
dfLists = [] features = []
print("get chunks from user data frames") print("get chunks from user data frames")
for i, user_flow in enumerate(get_flow_per_user(user_flow_df)): for i, user_flow in enumerate(get_flow_per_user(user_flow_df)):
(domainListsTmp, dfListsTmp) = get_user_chunks(user_flow, windowSize=windowSize, (domain_windows, feature_windows) = get_user_chunks(user_flow,
overlapping=True, maxLengthInSeconds=-1) windowSize=window_size,
domainLists += domainListsTmp overlapping=True,
dfLists += dfListsTmp maxLengthInSeconds=-1)
domains += domain_windows
features += feature_windows
# TODO: remove later # TODO: remove later
if i >= 10: if i >= 10:
break break
print("create training dataset") print("create training dataset")
return create_dataset_from_lists( return create_dataset_from_lists(
domains=domainLists, dfs=dfLists, vocab=char_dict, domains=domains, features=features, vocab=char_dict,
maxLen=maxLen, max_len=max_len,
use_cisco_features=use_cisco_features, urlSIPDIct=dict(), use_cisco_features=use_cisco_features, urlSIPDIct=dict(),
window_size=windowSize) window_size=window_size)
def create_dataset_from_lists(domains, dfs, vocab, maxLen, def create_dataset_from_lists(domains, features, vocab, max_len,
use_cisco_features=False, urlSIPDIct=dict(), use_cisco_features=False, urlSIPDIct=dict(),
window_size=10): window_size=10):
"""
combines domain and feature windows to sequential training data
:param domains: list of domain windows
:param features: list of feature windows
:param vocab:
:param max_len:
:param use_cisco_features: idk
:param urlSIPDIct: idk
:param window_size: size of the flow window
:return:
"""
# TODO: check for hits vs vth consistency # TODO: check for hits vs vth consistency
if 'hits' in dfs[0].keys(): # if 'hits' in dfs[0].keys():
hitName = 'hits' # hits_col = 'hits'
elif 'virusTotalHits' in dfs[0].keys(): # elif 'virusTotalHits' in dfs[0].keys():
hitName = 'virusTotalHits' # hits_col = 'virusTotalHits'
hits_col = "virusTotalHits"
numFlowFeatures = 3 numFlowFeatures = 3
numCiscoFeatures = 30 numCiscoFeatures = 30
numFeatures = numFlowFeatures numFeatures = numFlowFeatures
if use_cisco_features: if use_cisco_features:
numFeatures += numCiscoFeatures numFeatures += numCiscoFeatures
Xs = [] sample_size = len(domains)
hits = [] hits = []
names = [] names = []
servers = [] servers = []
trusted_hits = [] trusted_hits = []
for i in range(window_size):
Xs.append(np.zeros([len(domains), maxLen]))
Xs.append(np.zeros([len(domains), numFeatures]))
for i in tqdm(np.arange(len(domains)), miniters=10): domain_features = np.zeros((sample_size, window_size, max_len))
ctr = 0 flow_features = np.zeros((sample_size, window_size, numFeatures))
for j in range(np.min([window_size, len(domains[i])])):
Xs[ctr][i, :] = get_domain_features(domains[i][j], vocab, maxLen)
ctr += 1
if use_cisco_features:
Xs[ctr][i, 0:numFlowFeatures] = get_flow_features(dfs[i].iloc[j])
Xs[ctr][i, numFlowFeatures:] = get_cisco_features(dfs[i].iloc[j], urlSIPDIct)
else:
Xs[ctr][i, :] = get_flow_features(dfs[i].iloc[j])
ctr += 1
hits.append(np.max(dfs[i][hitName])) for i in tqdm(np.arange(sample_size), miniters=10):
names.append(np.unique(dfs[i]['user_hash'])) for j in range(window_size):
servers.append(np.max(dfs[i]['serverLabel'])) domain_features[i, j] = get_domain_features(domains[i][j], vocab, max_len)
trusted_hits.append(np.max(dfs[i]['trustedHits'])) flow_features[i, j] = get_flow_features(features[i].iloc[j])
return Xs, np.array(hits), np.array(names), np.array(servers), np.array(trusted_hits) # TODO: cisco features?
hits.append(np.max(features[i][hits_col]))
names.append(np.unique(features[i]['user_hash']))
servers.append(np.max(features[i]['serverLabel']))
trusted_hits.append(np.max(features[i]['trustedHits']))
X = [domain_features, flow_features]
return X, np.array(hits), np.array(names), np.array(servers), np.array(trusted_hits)
def discretize_label(values, threshold): def discretize_label(values, threshold):

View File

@ -104,10 +104,6 @@ def main():
client_labels = client_labels[idx] client_labels = client_labels[idx]
server_labels = server_tr[idx] server_labels = server_tr[idx]
# TODO: remove when features are flattened
for i in range(len(X_tr)):
X_tr[i] = X_tr[i][idx]
shared_cnn = models.get_shared_cnn(len(char_dict) + 1, embeddingSize, maxLen, shared_cnn = models.get_shared_cnn(len(char_dict) + 1, embeddingSize, maxLen,
domainFeatures, kernel_size, domainFeatures, 0.5) domainFeatures, kernel_size, domainFeatures, 0.5)