Changeset 1127 for framspy

Timestamp:

04/11/21 03:15:42 (4 years ago)

Author:

Maciej Komosinski

Message:

Experiment: measure running time and save checkpoints safely

File:

• framspy/evolalg/experiment.py (modified) (3 diffs)

framspy/evolalg/experiment.py

@@ +1 @@
+import os
 from typing import List, Callable, Union
 
 from evolalg.base.step import Step
 import pickle
+import time
 
 from evolalg.base.union_step import UnionStep
@@ -19 +21 @@
 
         self.init_population = init_population
+        self.running_time = 0
         self.step = StableGeneration(
             selection=selection,
@@ -47 +50 @@
     def run(self, num_generations):
         for i in range(self.generation + 1, num_generations + 1):
+            start_time = time.time()
             self.generation = i
             self.population = self.step(self.population)
             self.population = self.generation_modification(self.population)
 
+            self.running_time += time.time() - start_time
             if (self.checkpoint_path is not None
                     and self.checkpoint_interval is not None
                     and i % self.checkpoint_interval == 0):
-                with open(self.checkpoint_path, "wb") as file:
-                    pickle.dump(self, file)
+                self.save_checkpoint()
 
         self.population = self.end_steps(self.population)
+
+    def save_checkpoint(self):
+        tmp_filepath = self.checkpoint_path+"_tmp"
+        try:
+            with open(tmp_filepath, "wb") as file:
+                pickle.dump(self, file)
+            os.replace(tmp_filepath, self.checkpoint_path)  # ensures the new file was first saved OK (e.g. enough free space on device), then replace
+        except Exception as ex:
+            raise RuntimeError("Failed to save checkpoint '%s' (because: %s). This does not prevent the experiment from continuing, but let's stop here to fix the problem with saving checkpoints." % (tmp_filepath, ex))
+
 
     @staticmethod