Skip to content

Commit

Permalink
update StochasticAlgComparison.py
Browse files Browse the repository at this point in the history
  • Loading branch information
@emuskardin
emuskardin committed Jun 19, 2023
1 parent 69e1af1 commit fa21c3c
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 25 deletions.
14 changes: 9 additions & 5 deletions Benchmarking/StochasticAlgComparison.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@
max_walk_len=16, reset_after_cex=True)

learned_classic_mdp, data_mdp = run_stochastic_Lstar(input_alphabet, mdp_sul, eq_oracle, automaton_type='mdp',
min_rounds=10, strategy='classic', n_c=20, n_resample=500,
max_rounds=200, return_data=True, target_unambiguity=0.95,
min_rounds=10, strategy='classic', n_c=20, n_resample=2000,
stopping_range_dict={},
max_rounds=200, return_data=True, target_unambiguity=0.98,
print_level=1)

del mdp_sul
Expand All @@ -52,7 +53,7 @@

learned_smm, data_smm = run_stochastic_Lstar(input_alphabet, mdp_sul, eq_oracle, automaton_type='smm',
min_rounds=10, strategy='normal',
max_rounds=200, return_data=True, target_unambiguity=0.95,
max_rounds=200, return_data=True, target_unambiguity=0.98,
print_level=1)

smm_2_mdp = smm_to_mdp_conversion(learned_smm)
Expand All @@ -68,7 +69,7 @@
alergia_samples = []
for _ in range(num_alergia_samples):
sample = [mdp_sul.pre()]
for _ in range(random.randint(5, 25)):
for _ in range(random.randint(10, 30)):
action = random.choice(input_alphabet)
output = mdp_sul.step(action)
sample.append((action, output))
Expand All @@ -81,4 +82,7 @@

print('Classic MDP learning', mean(mdp_err.values()), mdp_err)
print('SMM learning', mean(smm_err.values()), smm_err)
print('Alergia learning', mean(alergia_error.values()), alergia_error)
print('Alergia learning', mean(alergia_error.values()), alergia_error)

print('Classic MDP traces', data_mdp["queries_learning"] + data_mdp["queries_eq_oracle"])
print('SMM learning traces', data_smm["queries_learning"] + data_smm["queries_eq_oracle"])
68 changes: 48 additions & 20 deletions Benchmarking/StopWithErorrRate.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import pickle
import random
import time
from collections import defaultdict
from statistics import mean

import aalpy.paths
Expand All @@ -13,7 +15,7 @@

path_to_dir = '../DotModels/MDPs/'
files = ['slot_machine.dot', 'bluetooth.dot'] #
files.reverse()
files = ['first_grid.dot', 'second_grid.dot', 'tcp.dot', 'mqtt.dot', 'bluetooth.dot', 'slot_machine.dot']

prop_folder = 'prism_eval_props/'

Expand All @@ -22,28 +24,54 @@

model_dict = {m.split('.')[0]: load_automaton_from_file(path_to_dir + m, automaton_type='mdp') for m in files}

model_type = ['mdp', 'smm']
model_type.reverse()
model_type = ['smm']
cex_processing = [None, 'longest_prefix', 'rs']
# model_type.reverse()

for file in files:
for mt in model_type:
exp_name = file.split('.')[0]
res = defaultdict(list)

print('--------------------------------------------------')
print('Experiment:', exp_name, )
# for file in files:
# for mt in model_type:
# for cp in cex_processing:
# for _ in range(4):
#
# exp_name = file.split('.')[0]
#
# print('--------------------------------------------------')
# print('Experiment:', exp_name, cp)
#
# original_mdp = model_dict[exp_name]
# input_alphabet = original_mdp.get_input_alphabet()
#
# mdp_sul = MdpSUL(original_mdp)
#
# eq_oracle = RandomWordEqOracle(input_alphabet, mdp_sul, num_walks=500, min_walk_len=5,
# max_walk_len=15, reset_after_cex=True)
#
# pbs = ((get_properties_file(exp_name),
# get_correct_prop_values(exp_name), 0.02 if exp_name != 'bluetooth' else 0.03))
# learned_classic_mdp, data_mdp = run_stochastic_Lstar(input_alphabet, mdp_sul, eq_oracle, automaton_type=mt,
# min_rounds=10,
# #property_based_stopping=pbs,
# cex_processing=cp,
# samples_cex_strategy=None,
# return_data=True, target_unambiguity=0.98,
# print_level=1)
#
# res[exp_name].append((cp, data_mdp['queries_learning'] + data_mdp['queries_eq_oracle']))

original_mdp = model_dict[exp_name]
input_alphabet = original_mdp.get_input_alphabet()
# with open('cex_processing_res.pickle', 'wb') as handle:
# pickle.dump(res, handle, protocol=pickle.HIGHEST_PROTOCOL)

mdp_sul = MdpSUL(original_mdp)
with open('cex_processing_res.pickle', 'rb') as handle:
res = pickle.load(handle)

eq_oracle = RandomWordEqOracle(input_alphabet, mdp_sul, num_walks=500, min_walk_len=5,
max_walk_len=20, reset_after_cex=True)
for key, val in res.items():
print(key)
sorted_by_cp = defaultdict(list)
for cp, data in val:
sorted_by_cp[cp].append(data)

pbs = ((get_properties_file(exp_name),
get_correct_prop_values(exp_name), 0.02))
learned_classic_mdp, data_mdp = run_stochastic_Lstar(input_alphabet, mdp_sul, eq_oracle, automaton_type=mt,
min_rounds=10,
property_based_stopping=pbs,
return_data=True, target_unambiguity=1.1,
print_level=2)
for cp_method, data in sorted_by_cp.items():
print(cp_method)
print(mean(data), min(data), max(data))

0 comments on commit fa21c3c

Please sign in to comment.