Expiry and defragmentation
This commit is contained in:
parent
537615e582
commit
f644027750
@ -199,6 +199,9 @@ class Commander(CommandLine):
|
||||
print(f'{num:5d} {nam}')
|
||||
print(f'{total:5d} total')
|
||||
|
||||
def do_defrag(self):
|
||||
self.store.defrag()
|
||||
|
||||
def do_waypoints(self, system_str=''):
|
||||
if system_str == '':
|
||||
if not self.has_ship(): return
|
||||
|
@ -95,13 +95,16 @@ class Store:
|
||||
def load(self):
|
||||
cnt = 0
|
||||
start_time = time()
|
||||
|
||||
total = 0
|
||||
free = 0
|
||||
self.fil.seek(0)
|
||||
offset = 0
|
||||
while (hdr := ChunkHeader.parse(self.fil)):
|
||||
# print(hdr)
|
||||
total += hdr.size
|
||||
if not hdr.in_use:
|
||||
self.fil.seek(hdr.size, 1)
|
||||
free += hdr.size
|
||||
continue
|
||||
data = self.fil.read(hdr.used)
|
||||
self.load_object(data, offset)
|
||||
@ -111,6 +114,7 @@ class Store:
|
||||
|
||||
dur = time() - start_time
|
||||
print(f'loaded {cnt} objects in {dur:.2f} seconds')
|
||||
print(f'Fragmented space: {free} / {total} bytes')
|
||||
|
||||
def allocate_chunk(self, sz):
|
||||
used = sz
|
||||
@ -127,6 +131,16 @@ class Store:
|
||||
h.write(self.fil)
|
||||
return offset, h
|
||||
|
||||
def purge(self, obj):
|
||||
if obj.file_offset is None:
|
||||
return
|
||||
self.fil.seek(obj.file_offset)
|
||||
hdr = ChunkHeader.parse(self.fil)
|
||||
hdr.in_use = False
|
||||
self.fil.seek(obj.file_offset)
|
||||
hdr.write(self.fil)
|
||||
obj.file_offset = None
|
||||
|
||||
def store(self, obj):
|
||||
data = self.dump_object(obj)
|
||||
osize = len(data)
|
||||
@ -207,6 +221,9 @@ class Store:
|
||||
typ = self.model_names[typ]
|
||||
|
||||
for m in self.data[typ].values():
|
||||
if m.is_expired():
|
||||
self.dirty(m)
|
||||
continue
|
||||
yield m
|
||||
|
||||
def all_members(self, system, typ=None):
|
||||
@ -219,10 +236,18 @@ class Store:
|
||||
if system not in self.system_members:
|
||||
return
|
||||
|
||||
garbage = set()
|
||||
for m in self.system_members[system]:
|
||||
if m.is_expired():
|
||||
self.dirty(m)
|
||||
garbage.add(m)
|
||||
continue
|
||||
if typ is None or type(m) == typ:
|
||||
yield m
|
||||
|
||||
for m in garbage:
|
||||
self.system_members[system].remove(m)
|
||||
|
||||
def cleanup(self):
|
||||
if time() < self.last_cleanup + self.cleanup_interval:
|
||||
return
|
||||
@ -233,8 +258,7 @@ class Store:
|
||||
if o.is_expired():
|
||||
expired.append(o)
|
||||
for o in expired:
|
||||
|
||||
# TODO
|
||||
self.purge(obj)
|
||||
|
||||
del self.data[type(o)][o.symbol]
|
||||
dur = time() - start_time
|
||||
@ -246,8 +270,21 @@ class Store:
|
||||
start_time = time()
|
||||
for obj in self.dirty_objects:
|
||||
it += 1
|
||||
if obj.is_expired():
|
||||
self.purge(obj)
|
||||
else:
|
||||
self.store(obj)
|
||||
self.fil.flush()
|
||||
self.dirty_objects = set()
|
||||
dur = time() - start_time
|
||||
# print(f'flush done {it} items {dur:.2f}')
|
||||
|
||||
def defrag(self):
|
||||
nm = self.fil.name
|
||||
self.fil.close()
|
||||
os.rename(nm, nm + '.bak')
|
||||
self.fil = open(nm, 'ab+')
|
||||
for t in self.data:
|
||||
for o in self.all(t):
|
||||
o.file_offset = None
|
||||
self.store(o)
|
||||
|
4
store.md
4
store.md
@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict
|
||||
* store.load(fil) loads all objects
|
||||
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
|
||||
* store.all(type) generator for all objects of a goven type
|
||||
* store.delete(typ, symbol)
|
||||
* store.cleanup() removes all expired objects
|
||||
* store.purge(obj)
|
||||
* store.clean() removes all expired objects
|
||||
* store.flush() writes all dirty objects to disk
|
||||
* store.defrag() consolidates the store file to minimize storage and loading time
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user