diff --git a/nullptr/commander.py b/nullptr/commander.py index 2f337e7..d0eeb3f 100644 --- a/nullptr/commander.py +++ b/nullptr/commander.py @@ -198,6 +198,9 @@ class Commander(CommandLine): total += num print(f'{num:5d} {nam}') print(f'{total:5d} total') + + def do_defrag(self): + self.store.defrag() def do_waypoints(self, system_str=''): if system_str == '': diff --git a/nullptr/store.py b/nullptr/store.py index ffb6dbf..188983f 100644 --- a/nullptr/store.py +++ b/nullptr/store.py @@ -95,13 +95,16 @@ class Store: def load(self): cnt = 0 start_time = time() - + total = 0 + free = 0 self.fil.seek(0) offset = 0 while (hdr := ChunkHeader.parse(self.fil)): # print(hdr) + total += hdr.size if not hdr.in_use: self.fil.seek(hdr.size, 1) + free += hdr.size continue data = self.fil.read(hdr.used) self.load_object(data, offset) @@ -111,6 +114,7 @@ class Store: dur = time() - start_time print(f'loaded {cnt} objects in {dur:.2f} seconds') + print(f'Fragmented space: {free} / {total} bytes') def allocate_chunk(self, sz): used = sz @@ -127,6 +131,16 @@ class Store: h.write(self.fil) return offset, h + def purge(self, obj): + if obj.file_offset is None: + return + self.fil.seek(obj.file_offset) + hdr = ChunkHeader.parse(self.fil) + hdr.in_use = False + self.fil.seek(obj.file_offset) + hdr.write(self.fil) + obj.file_offset = None + def store(self, obj): data = self.dump_object(obj) osize = len(data) @@ -207,6 +221,9 @@ class Store: typ = self.model_names[typ] for m in self.data[typ].values(): + if m.is_expired(): + self.dirty(m) + continue yield m def all_members(self, system, typ=None): @@ -219,10 +236,18 @@ class Store: if system not in self.system_members: return + garbage = set() for m in self.system_members[system]: + if m.is_expired(): + self.dirty(m) + garbage.add(m) + continue if typ is None or type(m) == typ: yield m + for m in garbage: + self.system_members[system].remove(m) + def cleanup(self): if time() < self.last_cleanup + self.cleanup_interval: return @@ -233,8 +258,7 @@ class Store: if o.is_expired(): expired.append(o) for o in expired: - - # TODO + self.purge(obj) del self.data[type(o)][o.symbol] dur = time() - start_time @@ -246,8 +270,21 @@ class Store: start_time = time() for obj in self.dirty_objects: it += 1 - self.store(obj) + if obj.is_expired(): + self.purge(obj) + else: + self.store(obj) self.fil.flush() self.dirty_objects = set() dur = time() - start_time # print(f'flush done {it} items {dur:.2f}') + + def defrag(self): + nm = self.fil.name + self.fil.close() + os.rename(nm, nm + '.bak') + self.fil = open(nm, 'ab+') + for t in self.data: + for o in self.all(t): + o.file_offset = None + self.store(o) diff --git a/store.md b/store.md index 978f340..733f457 100644 --- a/store.md +++ b/store.md @@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict * store.load(fil) loads all objects * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present * store.all(type) generator for all objects of a goven type -* store.delete(typ, symbol) -* store.cleanup() removes all expired objects +* store.purge(obj) +* store.clean() removes all expired objects * store.flush() writes all dirty objects to disk * store.defrag() consolidates the store file to minimize storage and loading time