rene/ma_cisco_malware
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add long final implementation

Browse Source
This commit is contained in:
René Knaebel 2017-11-05 22:52:50 +01:00
parent 14fef66a55
commit 7b8dfcebbe
4 changed files with 53 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hyperband.py
View File

@ -47,13 +47,13 @@ class Hyperband:
def try_params(self, n_iterations, params):
n_iterations = int(round(n_iterations))
embedding, model, new_model = models.get_models_by_params(params)
model = create_model(model, params["model_output"])
new_model = create_model(new_model, params["model_output"])
embedding, model, new_model, long_model = models.get_models_by_params(params)
if params["type"] in ("inter", "staggered"):
model = new_model
if params["type"] == "long":
model = long_model
model = create_model(model, params["model_output"])
callbacks = [EarlyStopping(monitor='val_loss',
patience=5,
@ -64,7 +64,7 @@ class Hyperband:
metrics=['accuracy'])
history = model.fit(self.X,
self.y,
self.y[0] if params["model_output"] == "client" else self.y,
batch_size=params["batch_size"],
epochs=n_iterations,
callbacks=callbacks,

18
main.py
View File

@ -182,12 +182,12 @@ def train(parameters, features, labels):
def load_data(data, domain_length, window_size, model_type):
# data preparation
domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(args.data,
args.data,
args.domain_length,
args.window)
domain_tr, flow_tr, name_tr, client_tr, server_windows_tr = dataset.load_or_generate_h5data(data,
data,
domain_length,
window_size)
server_tr = np.max(server_windows_tr, axis=1)
if args.model_type in ("inter", "staggered"):
if model_type in ("inter", "staggered"):
server_tr = np.expand_dims(server_windows_tr, 2)
return domain_tr, flow_tr, client_tr, server_tr
@ -246,13 +246,13 @@ def main_train(param=None):
custom_sample_weights = None
logger.info(f"Generator model with params: {param}")
embedding, model, new_model = models.get_models_by_params(param)
model = create_model(model, args.model_output)
new_model = create_model(new_model, args.model_output)
embedding, model, new_model, long_model = models.get_models_by_params(param)
if args.model_type in ("inter", "staggered"):
model = new_model
if args.model_type == "long":
model = long_model
model = create_model(model, args.model_output)
features = {"ipt_domains": domain_tr.value, "ipt_flows": flow_tr.value}
if args.model_output == "both":

11
models/__init__.py
View File

@ -37,13 +37,16 @@ def get_models_by_params(params: dict):
embedding_model = networks.get_embedding(embedding_size, domain_length, filter_embedding, kernel_embedding,
hidden_embedding, 0.5)
old_model = networks.get_model(0.25, flow_features, hidden_embedding, window_size, domain_length,
final = networks.get_model(0.25, flow_features, hidden_embedding, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
inter = networks.get_new_model(0.25, flow_features, hidden_embedding, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
new_model = networks.get_new_model(0.25, flow_features, hidden_embedding, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
long = networks.get_new_model2(0.25, flow_features, hidden_embedding, window_size, domain_length,
filter_main, kernel_main, dense_dim, embedding_model, model_output)
return embedding_model, old_model, new_model
return embedding_model, final, inter, long
def get_server_model_by_params(params: dict):

32
models/pauls_networks.py
View File

@ -103,3 +103,35 @@ def get_server_model(flow_features, domain_length, dense_dim, cnn):
out_server = Dense(1, activation="sigmoid", name="server")(y)
return KerasModel(inputs=[ipt_domains, ipt_flows], outputs=out_server)
def get_new_model2(dropout, flow_features, domain_features, window_size, domain_length, cnn_dims, kernel_size,
dense_dim, cnn, model_output="both") -> Model:
ipt_domains = Input(shape=(window_size, domain_length), name="ipt_domains")
ipt_flows = Input(shape=(window_size, flow_features), name="ipt_flows")
encoded = TimeDistributed(cnn, name="domain_cnn")(ipt_domains)
merged = keras.layers.concatenate([encoded, ipt_flows], -1)
y = Conv1D(cnn_dims,
kernel_size,
activation='relu')(merged)
# remove temporal dimension by global max pooling
y = GlobalMaxPooling1D()(y)
y = Dropout(dropout)(y)
y = Dense(dense_dim,
activation="relu",
name="dense_server")(y)
out_server = Dense(1, activation="sigmoid", name="server")(y)
# CNN processing a small slides of flow windows
y = Conv1D(cnn_dims,
kernel_size,
activation='relu')(merged)
# remove temporal dimension by global max pooling
y = GlobalMaxPooling1D()(y)
y = Dropout(dropout)(y)
y = Dense(dense_dim,
activation='relu',
name="dense_client")(y)
out_client = Dense(1, activation='sigmoid', name="client")(y)
return Model(ipt_domains, ipt_flows, out_client, out_server)