Expiry and defragmentation

nullptr/commander.py

@@ -199,6 +199,9 @@ class Commander(CommandLine):
       print(f'{num:5d} {nam}')
     print(f'{total:5d} total')
   
+  def do_defrag(self):
+    self.store.defrag()
+    
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return

nullptr/store.py

@@ -95,13 +95,16 @@ class Store:
   def load(self):
     cnt = 0
     start_time = time()
-    
+    total = 0
+    free = 0
     self.fil.seek(0)
     offset = 0
     while (hdr := ChunkHeader.parse(self.fil)):
       # print(hdr)
+      total += hdr.size
       if not hdr.in_use: 
         self.fil.seek(hdr.size, 1)
+        free += hdr.size
         continue
       data = self.fil.read(hdr.used)
       self.load_object(data, offset)
@@ -111,6 +114,7 @@ class Store:
       
     dur = time() - start_time
     print(f'loaded {cnt} objects in {dur:.2f} seconds')
+    print(f'Fragmented space: {free} / {total} bytes')
   
   def allocate_chunk(self, sz):
     used = sz
@@ -127,6 +131,16 @@ class Store:
     h.write(self.fil)
     return offset, h
     
+  def purge(self, obj):
+    if obj.file_offset is None:
+      return
+    self.fil.seek(obj.file_offset)
+    hdr = ChunkHeader.parse(self.fil)
+    hdr.in_use = False
+    self.fil.seek(obj.file_offset)
+    hdr.write(self.fil)
+    obj.file_offset = None
+    
   def store(self, obj):
     data = self.dump_object(obj)
     osize = len(data)
@@ -207,6 +221,9 @@ class Store:
       typ = self.model_names[typ]
     
     for m in self.data[typ].values():
+      if m.is_expired():
+        self.dirty(m)
+        continue
       yield m
   
   def all_members(self, system, typ=None):
@@ -219,10 +236,18 @@ class Store:
     if system not in self.system_members:
       return 
     
+    garbage = set()
     for m in self.system_members[system]:
+      if m.is_expired():
+        self.dirty(m)
+        garbage.add(m)
+        continue
       if typ is None or type(m) == typ:
         yield m
     
+    for m in garbage:
+      self.system_members[system].remove(m)
+      
   def cleanup(self):
     if time() < self.last_cleanup + self.cleanup_interval:
       return
@@ -233,8 +258,7 @@ class Store:
         if o.is_expired():
           expired.append(o)
     for o in expired:
-      
+      self.purge(obj)
-      # TODO
       
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
@@ -246,8 +270,21 @@ class Store:
     start_time = time()
     for obj in self.dirty_objects:
       it += 1
+      if obj.is_expired():
+        self.purge(obj)
+      else:
         self.store(obj)
     self.fil.flush()
     self.dirty_objects = set()
     dur = time() - start_time
     #  print(f'flush done {it} items {dur:.2f}')
+  
+  def defrag(self):
+    nm = self.fil.name
+    self.fil.close()
+    os.rename(nm, nm + '.bak')
+    self.fil = open(nm, 'ab+')
+    for t in self.data:
+      for o in self.all(t):
+        o.file_offset = None
+        self.store(o)

store.md

@@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict
 * store.load(fil) loads all objects
 * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
 * store.all(type) generator for all objects of a goven type
-* store.delete(typ, symbol)
+* store.purge(obj)
-* store.cleanup() removes all expired objects
+* store.clean() removes all expired objects
 * store.flush() writes all dirty objects to disk
 * store.defrag() consolidates the store file to minimize storage and loading time