Expiry and defragmentation

This commit is contained in:
Richard Bronkhorst 2023-07-14 12:33:31 +02:00
parent 537615e582
commit f644027750
3 changed files with 46 additions and 6 deletions

View File

@ -199,6 +199,9 @@ class Commander(CommandLine):
print(f'{num:5d} {nam}')
print(f'{total:5d} total')
def do_defrag(self):
self.store.defrag()
def do_waypoints(self, system_str=''):
if system_str == '':
if not self.has_ship(): return

View File

@ -95,13 +95,16 @@ class Store:
def load(self):
cnt = 0
start_time = time()
total = 0
free = 0
self.fil.seek(0)
offset = 0
while (hdr := ChunkHeader.parse(self.fil)):
# print(hdr)
total += hdr.size
if not hdr.in_use:
self.fil.seek(hdr.size, 1)
free += hdr.size
continue
data = self.fil.read(hdr.used)
self.load_object(data, offset)
@ -111,6 +114,7 @@ class Store:
dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds')
print(f'Fragmented space: {free} / {total} bytes')
def allocate_chunk(self, sz):
used = sz
@ -127,6 +131,16 @@ class Store:
h.write(self.fil)
return offset, h
def purge(self, obj):
if obj.file_offset is None:
return
self.fil.seek(obj.file_offset)
hdr = ChunkHeader.parse(self.fil)
hdr.in_use = False
self.fil.seek(obj.file_offset)
hdr.write(self.fil)
obj.file_offset = None
def store(self, obj):
data = self.dump_object(obj)
osize = len(data)
@ -207,6 +221,9 @@ class Store:
typ = self.model_names[typ]
for m in self.data[typ].values():
if m.is_expired():
self.dirty(m)
continue
yield m
def all_members(self, system, typ=None):
@ -219,10 +236,18 @@ class Store:
if system not in self.system_members:
return
garbage = set()
for m in self.system_members[system]:
if m.is_expired():
self.dirty(m)
garbage.add(m)
continue
if typ is None or type(m) == typ:
yield m
for m in garbage:
self.system_members[system].remove(m)
def cleanup(self):
if time() < self.last_cleanup + self.cleanup_interval:
return
@ -233,8 +258,7 @@ class Store:
if o.is_expired():
expired.append(o)
for o in expired:
# TODO
self.purge(obj)
del self.data[type(o)][o.symbol]
dur = time() - start_time
@ -246,8 +270,21 @@ class Store:
start_time = time()
for obj in self.dirty_objects:
it += 1
if obj.is_expired():
self.purge(obj)
else:
self.store(obj)
self.fil.flush()
self.dirty_objects = set()
dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}')
def defrag(self):
nm = self.fil.name
self.fil.close()
os.rename(nm, nm + '.bak')
self.fil = open(nm, 'ab+')
for t in self.data:
for o in self.all(t):
o.file_offset = None
self.store(o)

View File

@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict
* store.load(fil) loads all objects
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
* store.all(type) generator for all objects of a goven type
* store.delete(typ, symbol)
* store.cleanup() removes all expired objects
* store.purge(obj)
* store.clean() removes all expired objects
* store.flush() writes all dirty objects to disk
* store.defrag() consolidates the store file to minimize storage and loading time