test and fix the store

This commit is contained in:
Richard 2023-12-25 07:54:19 +01:00
parent 7038e8f852
commit bc8d565fc3
2 changed files with 123 additions and 12 deletions

View File

@ -57,7 +57,7 @@ class ChunkHeader:
return f'chunk {self.in_use} {self.size} {self.used}' return f'chunk {self.in_use} {self.size} {self.used}'
class Store: class Store:
def __init__(self, data_file): def __init__(self, data_file, verbose=False):
self.init_models() self.init_models()
self.fil = open_file(data_file) self.fil = open_file(data_file)
self.data = {m: {} for m in self.models} self.data = {m: {} for m in self.models}
@ -68,8 +68,18 @@ class Store:
self.slack = 0.1 self.slack = 0.1
self.slack_min = 64 self.slack_min = 64
self.slack_max = 1024 self.slack_max = 1024
self.verbose = verbose
self.load() self.load()
def p(self, m):
if not self.verbose:
return
print(m)
def close(self):
self.flush()
self.fil.close()
def init_models(self): def init_models(self):
self.models = all_subclasses(Base) self.models = all_subclasses(Base)
self.extensions = {c.ext(): c for c in self.models} self.extensions = {c.ext(): c for c in self.models}
@ -100,21 +110,21 @@ class Store:
self.fil.seek(0) self.fil.seek(0)
offset = 0 offset = 0
while (hdr := ChunkHeader.parse(self.fil)): while (hdr := ChunkHeader.parse(self.fil)):
# print(hdr) self.p(hdr)
total += hdr.size total += hdr.size
if not hdr.in_use: if not hdr.in_use:
self.fil.seek(hdr.size, 1) self.fil.seek(hdr.size, 1)
free += hdr.size free += hdr.size
continue else:
data = self.fil.read(hdr.used) data = self.fil.read(hdr.used)
self.load_object(data, offset) self.load_object(data, offset)
self.fil.seek(hdr.size - hdr.used, 1) self.fil.seek(hdr.size - hdr.used, 1)
offset = self.fil.tell()
cnt += 1 cnt += 1
offset = self.fil.tell()
dur = time() - start_time dur = time() - start_time
print(f'Loaded {cnt} objects in {dur:.2f} seconds') self.p(f'Loaded {cnt} objects in {dur:.2f} seconds')
print(f'Fragmented space: {free} / {total} bytes') self.p(f'Fragmented space: {free} / {total} bytes')
def allocate_chunk(self, sz): def allocate_chunk(self, sz):
used = sz used = sz
@ -174,8 +184,8 @@ class Store:
symbol = obj.symbol symbol = obj.symbol
obj.store = self obj.store = self
self.data[typ][symbol] = obj self.data[typ][symbol] = obj
if hasattr(obj, 'system') and obj.system != None: if hasattr(obj, 'system'):
system_str = obj.system.symbol system_str = obj.get_system().symbol
if system_str not in self.system_members: if system_str not in self.system_members:
self.system_members[system_str] = set() self.system_members[system_str] = set()
self.system_members[system_str].add(obj) self.system_members[system_str].add(obj)
@ -263,7 +273,7 @@ class Store:
del self.data[type(o)][o.symbol] del self.data[type(o)][o.symbol]
dur = time() - start_time dur = time() - start_time
print(f'cleaned {len(expired)} in {dur:.03f} seconds') self.p(f'cleaned {len(expired)} in {dur:.03f} seconds')
def flush(self): def flush(self):
self.cleanup() self.cleanup()

101
nullptr/test_store.py Normal file
View File

@ -0,0 +1,101 @@
import unittest
import tempfile
from nullptr.store import Store
from nullptr.models import Base
class Dummy(Base):
def define(self):
self.count: int = 0
self.data: str = ""
def update(self, d):
self.seta('count', d)
@classmethod
def ext(self):
return 'dum'
def f(self, detail=1):
r = super().f(detail)
if detail >2:
r += f' c:{self.count}'
return r
class TestStore(unittest.TestCase):
def setUp(self):
self.store_file = tempfile.NamedTemporaryFile()
self.s = Store(self.store_file.name, False)
def tearDown(self):
self.s.close()
self.store_file.close()
def reopen(self):
self.s.flush()
self.s.close()
self.s = Store(self.store_file.name, False)
def test_single(self):
dum = self.s.get(Dummy, "5", create=True)
dum.count = 1337
dum.data = "A" * 1000
self.reopen()
dum = self.s.get(Dummy, "5")
self.assertEqual(1337, dum.count)
def test_grow(self):
dum = self.s.get(Dummy, "5", create=True)
dum.data = "A"
dum2 = self.s.get(Dummy, "7",create=True)
self.reopen()
dum = self.s.get(Dummy, "5")
dum.data = "A" * 100
dum.count = 1337
self.reopen()
dum = self.s.get(Dummy, "5")
self.assertEqual(1337, dum.count)
def test_purge(self):
dum = self.s.get(Dummy, "5", create=True)
dum.data = "A"
dum2 = self.s.get(Dummy, "7",create=True)
dum2.count = 1337
self.s.flush()
self.s.purge(dum)
self.reopen()
dum2 = self.s.get(Dummy, "7")
self.assertEqual(1337, dum2.count)
def test_grow_last(self):
dum = self.s.get(Dummy, "5", create=True)
dum.data = "A"
dum2 = self.s.get(Dummy, "7",create=True)
self.reopen()
dum2 = self.s.get(Dummy, "7")
dum2.data = "A" * 100
dum2.count = 1337
dum3 = self.s.get(Dummy, "9",create=True)
dum3.count = 1338
self.reopen()
dum2 = self.s.get(Dummy, "7")
self.assertEqual(1337, dum2.count)
dum3 = self.s.get(Dummy, "9")
self.assertEqual(1338, dum3.count)
def test_purge_last(self):
dum = self.s.get(Dummy, "5", create=True)
dum.data = "A"
dum2 = self.s.get(Dummy, "7",create=True)
self.reopen()
dum2 = self.s.get(Dummy, "7")
self.s.purge(dum2)
dum3 = self.s.get(Dummy, "9",create=True)
dum3.count = 1338
self.reopen()
dum2 = self.s.get(Dummy, "7")
self.assertIsNone(dum2)
dum3 = self.s.get(Dummy, "9")
self.assertEqual(1338, dum3.count)