Skip to content

Commit

Permalink
Latest version
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackbergus
jackbergus committed Jun 19, 2024
1 parent 76a0cc2 commit e7fa53d
Show file tree
Hide file tree
Showing 20 changed files with 1,038 additions and 227 deletions.
74 changes: 0 additions & 74 deletions benchmark_model_size.csv
View file

This file was deleted.

10 changes: 10 additions & 0 deletions benchmark_poly.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,13 @@ filename_polyadic,mining_supp,reduction,reclassify,isFilenamePolyadic,cpp_prepro
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0.9,0,0,1,356.321,1007.08,131.226,56.9022,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0.9,1,0,0,328,911.844,128.183,57.1585,4.14329
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0.9,0,0,0,319.621,903.721,130.917,53.0961,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,1,300.86,800.527,117.078,20082.6,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,1,328.465,840.729,140.021,23425,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,1,293.664,774.624,116.497,19526.1,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,1,2886.19,9413,813.953,106920,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,1,0,1,2830.63,9298.7,568.097,110897,595.166
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,1,0,1,305.666,884.449,135.671,22015.3,175.412
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,1,0,1,331.926,843.255,136.441,21827.9,247.775
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,1,322.554,931.787,212.557,22430.9,0
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,1,0,0,308.326,856.088,150.196,23294.9,172.628
/home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json,0,0,0,0,303.417,828.267,130.524,22808.3,0
3 changes: 3 additions & 0 deletions expand_elements.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
find /home/giacomo/projects/knobab2_loggen/output_model_healthcare/debugged/ -maxdepth 1 -mindepth 1 -type d |while read fname; do
./cmake-build-release/knobab_json -f "$fname" -s 0.0 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json;
done
57 changes: 41 additions & 16 deletions include/knobab/mining/polyadic/polyadic_bolt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,11 @@ struct result_container {
if ((val == 1) || (val == -2))
val = 0;
}
for (size_t i = 0; i<total_log; i++) {
if (embeddings[i]==-2) {
embeddings[i] = 0;
}
}
}
};

Expand Down Expand Up @@ -956,7 +961,7 @@ struct polyadic_bolt {
Phi.erase(std::unique(Phi.begin(), Phi.end(), [](const pattern_mining_result<FastDatalessClause>& l, const pattern_mining_result<FastDatalessClause>& r) {
return std::tie(l.clause.casusu, l.clause.left, l.clause.right, l.clause.n) == std::tie(r.clause.casusu, r.clause.left, r.clause.right, r.clause.n);
}), Phi.end());
DEBUG_ASSERT(curr_size_Clauses == Phi.size());
// DEBUG_ASSERT(curr_size_Clauses == Phi.size());
}

inline void mdev(size_t i) {
Expand Down Expand Up @@ -1011,6 +1016,7 @@ struct polyadic_bolt {
}
setKnowledgeBaseAndInit(ptr);
std::vector<size_t> actLabels;
std::unordered_set<size_t> act_to_consider;
std::unordered_map<std::tuple<std::string,std::string,std::string>, std::vector<char>> result_map;
std::unordered_map<size_t, std::vector<size_t>> act_Labels;
std::unordered_map<size_t, std::vector<size_t>> noact_Labels;
Expand All @@ -1019,19 +1025,24 @@ struct polyadic_bolt {
ssize_t trace_id = -1;
size_t log_size = ptr->nTraces();
for (const auto& x : acts) {
size_t id = ptr->event_label_mapper.get(x);
actLabels.emplace_back(id);
auto a_beginend = kb->timed_dataless_exists(id);
auto& v = act_Labels[id];
trace_id = -1;
while (a_beginend.first != a_beginend.second) {
if (trace_id != a_beginend.first->entry.id.parts.trace_id) {
trace_id = a_beginend.first->entry.id.parts.trace_id;
v.emplace_back(trace_id);
if(ptr->event_label_mapper.signed_get(x)>0) {
size_t id = ptr->event_label_mapper.get(x);
actLabels.emplace_back(id);
act_to_consider.insert(id);
auto a_beginend = kb->timed_dataless_exists(id);
auto& v = act_Labels[id];
trace_id = -1;
while (a_beginend.first != a_beginend.second) {
if (trace_id != a_beginend.first->entry.id.parts.trace_id) {
trace_id = a_beginend.first->entry.id.parts.trace_id;
v.emplace_back(trace_id);
}
a_beginend.first++;
}
a_beginend.first++;
set_complement(log_size, v.begin(), v.end(), std::back_inserter(noact_Labels[id]));
} else {
// std::cerr << x << std::endl;
}
set_complement(log_size, v.begin(), v.end(), std::back_inserter(noact_Labels[id]));
}
// remove_duplicates(actLabels);
FastDatalessClause clause;
Expand All @@ -1045,16 +1056,19 @@ struct polyadic_bolt {
for (size_t trace_id = 0; trace_id < log_size; trace_id++) {
const auto& first_last =kb->act_table_by_act_id.secondary_index.at(trace_id);
for (auto it = first_last.first->begin(), en = first_last.first->end(); it!=en; it++) {
first[it->first].emplace_back(trace_id);
if (act_to_consider.contains(it->first))
first[it->first].emplace_back(trace_id);
}
for (auto it = first_last.second->begin(), en = first_last.second->end(); it!=en; it++) {
last[it->first].emplace_back(trace_id);
if (act_to_consider.contains(it->first))
last[it->first].emplace_back(trace_id);
}
}
std::cout << "First..." << std::endl;
std::tuple<std::string,std::string,std::string> simplistic_clause{"Init","","§1"};
for (const auto& [act_id, traces] : first) {
all_VIOL.clear();
if (!act_to_consider.contains(act_id)) continue;
std::get<1>(simplistic_clause) = ptr->event_label_mapper.get(act_id);
auto& v = result_map[simplistic_clause];
v.resize(log_size, -1);
Expand All @@ -1066,6 +1080,7 @@ struct polyadic_bolt {
std::cout << "Last..." << std::endl;
for (const auto& [act_id, traces] : last) {
all_VIOL.clear();
if (!act_to_consider.contains(act_id)) continue;
std::get<1>(simplistic_clause) = ptr->event_label_mapper.get(act_id);
auto& v = result_map[simplistic_clause];
v.resize(log_size, -1);
Expand All @@ -1078,10 +1093,15 @@ struct polyadic_bolt {
std::get<0>(simplistic_clause) = "Exists";
std::cout << "Exists..." << std::endl;
for (const auto& [act_id, countings] : exists) {
if(ptr->event_label_mapper.signed_get(act_id)<0) {
continue;
}
if (!act_to_consider.contains(ptr->event_label_mapper.get(act_id))) continue;
std::get<1>(simplistic_clause) = act_id;
auto indexes = ptr->resolveCountingData(act_id);
if ((indexes.first == indexes.second) && (indexes.first == (uint32_t)-1)) {
exit(5);
continue;
// exit(5);
} else {
std::unordered_map<size_t, std::string> MAP;
for (size_t count : countings) {
Expand All @@ -1102,10 +1122,15 @@ struct polyadic_bolt {
std::get<0>(simplistic_clause) = "Absence";
std::cout << "Absence..." << std::endl;
for (const auto& [act_id, countings] : absence) {
if(ptr->event_label_mapper.signed_get(act_id)<0) {
continue;
}
if (!act_to_consider.contains(ptr->event_label_mapper.get(act_id))) continue;
std::get<1>(simplistic_clause) = act_id;
auto indexes = ptr->resolveCountingData(act_id);
if ((indexes.first == indexes.second) && (indexes.first == (uint32_t)-1)) {
exit(4);
continue;
// exit(4);
} else {
std::unordered_map<size_t, std::string> MAP;
for (size_t count : countings) {
Expand Down
1 change: 1 addition & 0 deletions new_pipeline.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ struct benchmarking {
#include <yaucl/strings/string_utils.h>

int main(int argc, char **argv) {
// Phases 01 (mining the models from the data) and 03 (deriving the decision tree structure)

// CyberSecurity configuration:
// -s 0.8 --nonPoly=tab --nonPoly=tab --nonPoly=tab --nonPoly=tab --nonPoly=tab --nonPoly=tab --nonPoly=tab --nonPoly=tab /home/giacomo/Scaricati/classes/Adware.tab_100.tab /home/giacomo/Scaricati/classes/Backdoor.tab_100.tab /home/giacomo/Scaricati/classes/Downloader.tab_100.tab /home/giacomo/Scaricati/classes/Dropper.tab_100.tab /home/giacomo/Scaricati/classes/Spyware.tab_100.tab /home/giacomo/Scaricati/classes/Trojan.tab_100.tab /home/giacomo/Scaricati/classes/Virus.tab_100.tab /home/giacomo/Scaricati/classes/Worms.tab_100.tab
Expand Down
6 changes: 5 additions & 1 deletion poly_bench.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,8 @@ done
./cmake-build-release/knobab_json -s 0.9 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -r -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.9 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.9 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -r -l -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.9 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -l -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.9 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -l -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.0 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -r -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.0 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.0 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -r -l -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
./cmake-build-release/knobab_json -s 0.0 -d user -i day -i span -i "__class" -i "__label" -i time -i fulltime -l -p /home/giacomo/projects/sdd-processing/sdd-processing/log_weekly.json
Loading

0 comments on commit e7fa53d

Please sign in to comment.