Expiry and defragmentation
This commit is contained in:
parent
537615e582
commit
f644027750
@ -199,6 +199,9 @@ class Commander(CommandLine):
|
|||||||
print(f'{num:5d} {nam}')
|
print(f'{num:5d} {nam}')
|
||||||
print(f'{total:5d} total')
|
print(f'{total:5d} total')
|
||||||
|
|
||||||
|
def do_defrag(self):
|
||||||
|
self.store.defrag()
|
||||||
|
|
||||||
def do_waypoints(self, system_str=''):
|
def do_waypoints(self, system_str=''):
|
||||||
if system_str == '':
|
if system_str == '':
|
||||||
if not self.has_ship(): return
|
if not self.has_ship(): return
|
||||||
|
@ -95,13 +95,16 @@ class Store:
|
|||||||
def load(self):
|
def load(self):
|
||||||
cnt = 0
|
cnt = 0
|
||||||
start_time = time()
|
start_time = time()
|
||||||
|
total = 0
|
||||||
|
free = 0
|
||||||
self.fil.seek(0)
|
self.fil.seek(0)
|
||||||
offset = 0
|
offset = 0
|
||||||
while (hdr := ChunkHeader.parse(self.fil)):
|
while (hdr := ChunkHeader.parse(self.fil)):
|
||||||
# print(hdr)
|
# print(hdr)
|
||||||
|
total += hdr.size
|
||||||
if not hdr.in_use:
|
if not hdr.in_use:
|
||||||
self.fil.seek(hdr.size, 1)
|
self.fil.seek(hdr.size, 1)
|
||||||
|
free += hdr.size
|
||||||
continue
|
continue
|
||||||
data = self.fil.read(hdr.used)
|
data = self.fil.read(hdr.used)
|
||||||
self.load_object(data, offset)
|
self.load_object(data, offset)
|
||||||
@ -111,6 +114,7 @@ class Store:
|
|||||||
|
|
||||||
dur = time() - start_time
|
dur = time() - start_time
|
||||||
print(f'loaded {cnt} objects in {dur:.2f} seconds')
|
print(f'loaded {cnt} objects in {dur:.2f} seconds')
|
||||||
|
print(f'Fragmented space: {free} / {total} bytes')
|
||||||
|
|
||||||
def allocate_chunk(self, sz):
|
def allocate_chunk(self, sz):
|
||||||
used = sz
|
used = sz
|
||||||
@ -127,6 +131,16 @@ class Store:
|
|||||||
h.write(self.fil)
|
h.write(self.fil)
|
||||||
return offset, h
|
return offset, h
|
||||||
|
|
||||||
|
def purge(self, obj):
|
||||||
|
if obj.file_offset is None:
|
||||||
|
return
|
||||||
|
self.fil.seek(obj.file_offset)
|
||||||
|
hdr = ChunkHeader.parse(self.fil)
|
||||||
|
hdr.in_use = False
|
||||||
|
self.fil.seek(obj.file_offset)
|
||||||
|
hdr.write(self.fil)
|
||||||
|
obj.file_offset = None
|
||||||
|
|
||||||
def store(self, obj):
|
def store(self, obj):
|
||||||
data = self.dump_object(obj)
|
data = self.dump_object(obj)
|
||||||
osize = len(data)
|
osize = len(data)
|
||||||
@ -207,6 +221,9 @@ class Store:
|
|||||||
typ = self.model_names[typ]
|
typ = self.model_names[typ]
|
||||||
|
|
||||||
for m in self.data[typ].values():
|
for m in self.data[typ].values():
|
||||||
|
if m.is_expired():
|
||||||
|
self.dirty(m)
|
||||||
|
continue
|
||||||
yield m
|
yield m
|
||||||
|
|
||||||
def all_members(self, system, typ=None):
|
def all_members(self, system, typ=None):
|
||||||
@ -219,10 +236,18 @@ class Store:
|
|||||||
if system not in self.system_members:
|
if system not in self.system_members:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
garbage = set()
|
||||||
for m in self.system_members[system]:
|
for m in self.system_members[system]:
|
||||||
|
if m.is_expired():
|
||||||
|
self.dirty(m)
|
||||||
|
garbage.add(m)
|
||||||
|
continue
|
||||||
if typ is None or type(m) == typ:
|
if typ is None or type(m) == typ:
|
||||||
yield m
|
yield m
|
||||||
|
|
||||||
|
for m in garbage:
|
||||||
|
self.system_members[system].remove(m)
|
||||||
|
|
||||||
def cleanup(self):
|
def cleanup(self):
|
||||||
if time() < self.last_cleanup + self.cleanup_interval:
|
if time() < self.last_cleanup + self.cleanup_interval:
|
||||||
return
|
return
|
||||||
@ -233,8 +258,7 @@ class Store:
|
|||||||
if o.is_expired():
|
if o.is_expired():
|
||||||
expired.append(o)
|
expired.append(o)
|
||||||
for o in expired:
|
for o in expired:
|
||||||
|
self.purge(obj)
|
||||||
# TODO
|
|
||||||
|
|
||||||
del self.data[type(o)][o.symbol]
|
del self.data[type(o)][o.symbol]
|
||||||
dur = time() - start_time
|
dur = time() - start_time
|
||||||
@ -246,8 +270,21 @@ class Store:
|
|||||||
start_time = time()
|
start_time = time()
|
||||||
for obj in self.dirty_objects:
|
for obj in self.dirty_objects:
|
||||||
it += 1
|
it += 1
|
||||||
self.store(obj)
|
if obj.is_expired():
|
||||||
|
self.purge(obj)
|
||||||
|
else:
|
||||||
|
self.store(obj)
|
||||||
self.fil.flush()
|
self.fil.flush()
|
||||||
self.dirty_objects = set()
|
self.dirty_objects = set()
|
||||||
dur = time() - start_time
|
dur = time() - start_time
|
||||||
# print(f'flush done {it} items {dur:.2f}')
|
# print(f'flush done {it} items {dur:.2f}')
|
||||||
|
|
||||||
|
def defrag(self):
|
||||||
|
nm = self.fil.name
|
||||||
|
self.fil.close()
|
||||||
|
os.rename(nm, nm + '.bak')
|
||||||
|
self.fil = open(nm, 'ab+')
|
||||||
|
for t in self.data:
|
||||||
|
for o in self.all(t):
|
||||||
|
o.file_offset = None
|
||||||
|
self.store(o)
|
||||||
|
4
store.md
4
store.md
@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict
|
|||||||
* store.load(fil) loads all objects
|
* store.load(fil) loads all objects
|
||||||
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
|
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
|
||||||
* store.all(type) generator for all objects of a goven type
|
* store.all(type) generator for all objects of a goven type
|
||||||
* store.delete(typ, symbol)
|
* store.purge(obj)
|
||||||
* store.cleanup() removes all expired objects
|
* store.clean() removes all expired objects
|
||||||
* store.flush() writes all dirty objects to disk
|
* store.flush() writes all dirty objects to disk
|
||||||
* store.defrag() consolidates the store file to minimize storage and loading time
|
* store.defrag() consolidates the store file to minimize storage and loading time
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user