test and fix the store

nullptr/store.py

@@ -57,7 +57,7 @@ class ChunkHeader:
     return f'chunk {self.in_use} {self.size} {self.used}'
     
 class Store:
-  def __init__(self, data_file):
+  def __init__(self, data_file, verbose=False):
     self.init_models()
     self.fil = open_file(data_file)
     self.data = {m: {} for m in self.models}
@@ -68,8 +68,18 @@ class Store:
     self.slack = 0.1
     self.slack_min = 64
     self.slack_max = 1024
+    self.verbose = verbose
     self.load()
   
+  def p(self,  m):
+    if not self.verbose:
+      return
+    print(m)
+  
+  def close(self):
+    self.flush()
+    self.fil.close()
+  
   def init_models(self):
     self.models = all_subclasses(Base)
     self.extensions = {c.ext(): c for c in self.models}
@@ -100,21 +110,21 @@ class Store:
     self.fil.seek(0)
     offset = 0
     while (hdr := ChunkHeader.parse(self.fil)):
-      # print(hdr)
+      self.p(hdr)
       total += hdr.size
       if not hdr.in_use: 
         self.fil.seek(hdr.size, 1)
         free += hdr.size
-        continue
+      else:
-      data = self.fil.read(hdr.used)
+        data = self.fil.read(hdr.used)
-      self.load_object(data, offset)
+        self.load_object(data, offset)
-      self.fil.seek(hdr.size - hdr.used, 1)
+        self.fil.seek(hdr.size - hdr.used, 1)
+        cnt += 1
       offset = self.fil.tell()
-      cnt += 1
       
     dur = time() - start_time
-    print(f'Loaded {cnt} objects in {dur:.2f} seconds')
+    self.p(f'Loaded {cnt} objects in {dur:.2f} seconds')
-    print(f'Fragmented space: {free} / {total} bytes')
+    self.p(f'Fragmented space: {free} / {total} bytes')
   
   def allocate_chunk(self, sz):
     used = sz
@@ -174,8 +184,8 @@ class Store:
     symbol = obj.symbol
     obj.store = self
     self.data[typ][symbol] = obj
-    if hasattr(obj, 'system') and obj.system != None:
+    if hasattr(obj, 'system'):
-      system_str = obj.system.symbol
+      system_str = obj.get_system().symbol
       if system_str not in self.system_members:
         self.system_members[system_str] = set()
       self.system_members[system_str].add(obj)
@@ -263,7 +273,7 @@ class Store:
       
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
-    print(f'cleaned {len(expired)} in {dur:.03f} seconds')
+    self.p(f'cleaned {len(expired)} in {dur:.03f} seconds')
       
   def flush(self):
     self.cleanup()

nullptr/test_store.py

@@ -0,0 +1,101 @@
+import unittest
+import tempfile
+from nullptr.store import Store
+from nullptr.models import Base
+
+class Dummy(Base):
+  def define(self):
+    self.count: int = 0
+    self.data: str = ""
+  
+  def update(self, d):
+    self.seta('count', d)
+    
+  @classmethod
+  def ext(self):
+    return 'dum'
+  
+  def f(self, detail=1):
+    r = super().f(detail)
+    if detail >2:
+      r += f' c:{self.count}'
+    return r
+    
+class TestStore(unittest.TestCase):
+  def setUp(self):
+    self.store_file = tempfile.NamedTemporaryFile()
+    self.s = Store(self.store_file.name, False)
+  
+  def tearDown(self):
+    self.s.close()
+    self.store_file.close()
+    
+  def reopen(self):
+    self.s.flush()
+    self.s.close()
+    self.s = Store(self.store_file.name, False)
+    
+  def test_single(self):
+    dum = self.s.get(Dummy, "5", create=True)
+    dum.count = 1337
+    dum.data = "A" * 1000
+    self.reopen()
+    
+    dum = self.s.get(Dummy, "5")
+    self.assertEqual(1337, dum.count)
+  
+  def test_grow(self):
+    dum = self.s.get(Dummy, "5", create=True)
+    dum.data = "A"
+    dum2 = self.s.get(Dummy, "7",create=True)
+    self.reopen()
+    dum = self.s.get(Dummy, "5")
+    dum.data = "A" * 100
+    dum.count = 1337
+    self.reopen()
+    dum = self.s.get(Dummy, "5")
+    self.assertEqual(1337, dum.count)
+  
+  def test_purge(self):
+    dum = self.s.get(Dummy, "5", create=True)
+    dum.data = "A"
+    dum2 = self.s.get(Dummy, "7",create=True)
+    dum2.count = 1337
+    self.s.flush()
+    self.s.purge(dum)
+    self.reopen()
+    dum2 = self.s.get(Dummy, "7")
+    self.assertEqual(1337, dum2.count)
+  
+  def test_grow_last(self):
+    dum = self.s.get(Dummy, "5", create=True)
+    dum.data = "A"
+    dum2 = self.s.get(Dummy, "7",create=True)
+    self.reopen()
+    dum2 = self.s.get(Dummy, "7")
+    dum2.data = "A" * 100
+    dum2.count = 1337
+    dum3 = self.s.get(Dummy, "9",create=True)
+    dum3.count = 1338
+    self.reopen()
+    dum2 = self.s.get(Dummy, "7")
+    self.assertEqual(1337, dum2.count)
+    dum3 = self.s.get(Dummy, "9")
+    self.assertEqual(1338, dum3.count)
+    
+  def test_purge_last(self):
+    dum = self.s.get(Dummy, "5", create=True)
+    dum.data = "A"
+    dum2 = self.s.get(Dummy, "7",create=True)
+    self.reopen()
+    dum2 = self.s.get(Dummy, "7")
+    self.s.purge(dum2)
+    dum3 = self.s.get(Dummy, "9",create=True)
+    dum3.count = 1338
+    self.reopen()
+    dum2 = self.s.get(Dummy, "7")
+    self.assertIsNone(dum2)
+    dum3 = self.s.get(Dummy, "9")
+    self.assertEqual(1338, dum3.count)
+    
+