source: framspy/evolalg/tests/test_diferent_settings.py @ 1224

Last change on this file since 1224 was 1205, checked in by Maciej Komosinski, 21 months ago

Splitting (potentially) multiple .sim files is already done by FramsticksLib?.py
File size: 9.5 KB
Line 
1import argparse
2import sys
3
4from FramsticksLib import FramsticksLib
5
6from ..frams_base.experiment_frams_niching import ExperimentFramsNiching
7from ..frams_base.experiment_frams_islands import ExperimentFramsIslands
8from ..numerical_example.numerical_example import ExperimentNumerical
9from ..numerical_example.numerical_islands_example import ExperimentNumericalIslands
10
11
12from ..utils import ensureDir
13
14SETTINGS_TO_TEST_NUMERIC = {
15    'hof_size': [0, 10],
16    'popsize': [8],
17    'archive': [8],
18    'pmut': [0.7],
19    'pxov': [0.2],
20    'tournament': [5],
21    'initialgenotype':[[100, 100, 100, 100], [-100,-100]]
22}
23
24SETTINGS_TO_TEST_NUMERIC_ISLAND = {
25    'hof_size': [0, 10],
26    'popsize': [8],
27    'archive': [8],
28    'pmut': [0.7],
29    'pxov': [0.2],
30    'tournament': [5],
31    'migration_interval': [1,5],
32    'number_of_populations':[1,5],
33    'initialgenotype':[[100, 100, 100, 100], [-100,-100]]
34}
35
36SETTINGS_TO_TEST_FRAMS_NICHING = {
37    'opt': ['velocity', 'vertpos'],
38    'max_numparts': [None],
39    'max_numjoints': [20],
40    'max_numneurons': [20],
41    'max_numconnections': [None],
42    'max_numgenochars': [20],
43    'hof_size': [0, 10],
44    'normalize': ['none', 'max', 'sum'],
45    'dissim': [-2, -1, 1, 2],
46    'fit': ['niching', 'novelty', 'nsga2', 'nslc', 'raw'],
47    'genformat': ['1'],
48    'popsize': [8],
49    'archive': [8],
50    'initialgenotype': [None],
51    'pmut': [0.7],
52    'pxov': [0.2],
53    'tournament': [5]
54}
55
56SETTINGS_TO_TEST_FRAMS_ISLANDS = {
57    'opt': ['velocity', 'vertpos'],
58    'max_numparts': [None],
59    'max_numjoints': [20],
60    'max_numneurons': [20],
61    'max_numconnections': [None],
62    'max_numgenochars': [20],
63    'hof_size': [0, 10],
64    'migration_interval': [1,5],
65    'number_of_populations':[1,5],
66    'genformat': ['1'],
67    'popsize': [8],
68    'initialgenotype': [None],
69    'pmut': [0.7],
70    'pxov': [0.2],
71    'tournament': [5]
72}
73
74def test_run_experiment_numerical(params):
75    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
76
77    experiment = ExperimentNumerical(
78                                    hof_size=params['hof_size'],
79                                    popsize=params['popsize'],
80                                    save_only_best=True,)
81
82    experiment.evolve(hof_savefile=None,
83                      generations=2,
84                      initialgenotype=params['initialgenotype'],
85                      pmut=params['pmut'],
86                      pxov=params['pxov'],
87                      tournament_size=params['tournament'])
88
89
90def test_run_experiment_numerical_islands(params):
91    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
92
93    experiment = ExperimentNumericalIslands(hof_size=params['hof_size'],
94                                    popsize=params['popsize'],
95                                    save_only_best=True,
96                                    migration_interval=params['migration_interval'],
97                                    number_of_populations=params['number_of_populations'])
98
99
100    experiment.evolve(hof_savefile=None,
101                      generations=2,
102                      initialgenotype=params['initialgenotype'],
103                      pmut=params['pmut'],
104                      pxov=params['pxov'],
105                      tournament_size=params['tournament'])
106
107def test_run_experiment_frams_niching(params):
108    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
109    opt_criteria = params['opt'].split(",")
110    framsLib = FramsticksLib(
111        parsed_args.path, parsed_args.lib, parsed_args.sim)
112    constrains = {"max_numparts": params['max_numparts'],
113                  "max_numjoints": params['max_numjoints'],
114                  "max_numneurons": params['max_numneurons'],
115                  "max_numconnections": params['max_numconnections'],
116                  "max_numgenochars": params['max_numgenochars'],
117                  }
118
119    print('Best individuals:')
120    experiment = ExperimentFramsNiching(frams_lib=framsLib,
121                                        optimization_criteria=opt_criteria,
122                                        hof_size=params['hof_size'],
123                                        constraints=constrains,
124                                        normalize=params['normalize'],
125                                        dissim=params['dissim'],
126                                        fit=params['fit'],
127                                        genformat=params['genformat'],
128                                        popsize=params['popsize'],
129                                        archive_size=params['archive'],
130                                        save_only_best=True)
131
132    experiment.evolve(hof_savefile=None,
133                      generations=2,
134                      initialgenotype=params['initialgenotype'],
135                      pmut=params['pmut'],
136                      pxov=params['pxov'],
137                      tournament_size=params['tournament'])
138
139def test_run_experiment_frams_island(params):
140    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
141    opt_criteria = params['opt'].split(",")
142    framsLib = FramsticksLib(
143        parsed_args.path, parsed_args.lib, parsed_args.sim)
144    constrains = {"max_numparts": params['max_numparts'],
145                  "max_numjoints": params['max_numjoints'],
146                  "max_numneurons": params['max_numneurons'],
147                  "max_numconnections": params['max_numconnections'],
148                  "max_numgenochars": params['max_numgenochars'],
149                  }
150
151    print('Best individuals:')
152    experiment = ExperimentFramsIslands(frams_lib=framsLib,
153                                        optimization_criteria=opt_criteria,
154                                        hof_size=params['hof_size'],
155                                        constraints=constrains,
156                                        genformat=params['genformat'],
157                                        popsize=params['popsize'],
158                                        migration_interval=params['migration_interval'],
159                                        number_of_populations=params['number_of_populations'],
160                                        save_only_best=True)
161
162    experiment.evolve(hof_savefile=None,
163                      generations=2,
164                      initialgenotype=params['initialgenotype'],
165                      pmut=params['pmut'],
166                      pxov=params['pxov'],
167                      tournament_size=params['tournament'])
168
169
170def parseArguments():
171    parser = argparse.ArgumentParser(
172        description='Run this program with "python -u %s" if you want to disable buffering of its output.' % sys.argv[0])
173    parser.add_argument('-path', type=ensureDir, required=True,
174                        help='Path to Framsticks CLI without trailing slash.')
175    parser.add_argument('-lib', required=False,
176                        help='Library name. If not given, "frams-objects.dll" or "frams-objects.so" is assumed depending on the platform.')
177    parser.add_argument('-sim', required=False, default="eval-allcriteria.sim",
178                        help="The name of the .sim file with settings for evaluation, mutation, crossover, and similarity estimation. If not given, \"eval-allcriteria.sim\" is assumed by default. Must be compatible with the \"standard-eval\" expdef. If you want to provide more files, separate them with a semicolon ';'.")
179
180    return parser.parse_args()
181
182
183def get_params_sets(settings):
184    params_sets = []
185    for k in settings.keys():
186        temp_param_set = []
187        for value in settings[k]:
188            if params_sets:
189                for exsiting_set in params_sets:
190                    copy_of_set = exsiting_set.copy()
191                    copy_of_set[k] = value
192                    temp_param_set.append(copy_of_set)
193            else:
194                temp_param_set.append({k: value})
195        params_sets = temp_param_set
196    return params_sets
197
198
199def cover_to_test(params, run_exp):
200    try:
201        run_exp(params)
202        return [1, None]
203    except Exception as e:
204        return [0, f"Experiment {run_exp.__name__} with params:{params} failied with the stack:{e}"]
205
206
207def run_tests():
208    result = []
209    print("TESTING NUMERICAL")
210    params_sets = get_params_sets(SETTINGS_TO_TEST_NUMERIC)
211    print(f"Starting executing {len(params_sets)} experiments")
212    result.extend([cover_to_test(params,test_run_experiment_numerical) for params in params_sets])
213
214    print("TESTING NUMERICAL ISLANDS")
215    params_sets = get_params_sets(SETTINGS_TO_TEST_NUMERIC_ISLAND)
216    print(f"Starting executing {len(params_sets)} experiments")
217    result.extend([cover_to_test(params,test_run_experiment_numerical_islands) for params in params_sets])
218
219    print("TESTING FRAMS NICHING")
220    params_sets = get_params_sets(SETTINGS_TO_TEST_FRAMS_NICHING)
221    print(f"Starting executing {len(params_sets)} experiments")
222    result.extend([cover_to_test(params, test_run_experiment_frams_niching) for params in params_sets])
223
224    print("TESTING FRAMS ISLANDS")
225    params_sets = get_params_sets(SETTINGS_TO_TEST_FRAMS_ISLANDS)
226    print(f"Starting executing {len(params_sets)} experiments")
227    result.extend([cover_to_test(params,test_run_experiment_frams_island) for params in params_sets])
228
229    Successful = sum([1 for r in result  if r[0] == 1])
230    Failed = sum([1 for r in result if r[0] == 0])
231    print(f"{Successful} out of {len(result)} passed")
232    if Successful < Failed:
233        print("Experiments that failed :{Failed}")
234        [print(r[1]) for r in result if r[1]]
235
236
237if __name__ == "__main__":
238    parsed_args = parseArguments()
239    run_tests()
Note: See TracBrowser for help on using the repository browser.