New store setup

Dockerfile

@@ -9,4 +9,4 @@ ADD --chown=user . /app
 RUN chmod +x /app/main.py
 VOLUME /data
 ENTRYPOINT [ "python3", "/app/main.py"]
-CMD ["-s", "/data/"]
+CMD ["-s", "/data/store.npt"]

main.py

@@ -1,8 +1,8 @@
 import argparse
 from nullptr.commander import Commander
-
+from nullptr.models.base import Base
 def main(args):
-  c = Commander(args.store_dir)
+  c = Commander(args.store_file)
   c.run()
   
 # X1-AG74-41076A
@@ -10,6 +10,6 @@ def main(args):
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--store-dir', default='data')
+  parser.add_argument('-s', '--store-file', default='data/store.npt')
   args = parser.parse_args()
   main(args)

nullptr/api.py

@@ -69,6 +69,7 @@ class Api:
     }
     result = self.request('post', 'register', data, need_token=False)
     token = mg(result, 'token')
+    self.agent.update(mg(result, 'agent'))
     self.agent.token = token
   
   def info(self):
@@ -87,13 +88,12 @@ class Api:
     return self.store.update_list(Waypoint, data)
   
   def marketplace(self, waypoint):
-    system = waypoint.system()
+    system = waypoint.system
-    symbol = str(waypoint)
     data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
     return self.store.update(Marketplace, data)
   
   def jumps(self, waypoint):
-    data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
+    data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
     symbol = str(waypoint)
     return self.store.update(Jumpgate, data, symbol)
   

nullptr/commander.py

@@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine
 from nullptr.store import Store
 from nullptr.analyzer import  Analyzer
 import argparse
-from nullptr.models.agent import Agent
+from nullptr.models import *
-from nullptr.models.system import System
-from nullptr.models.waypoint import Waypoint
-from nullptr.models.marketplace import Marketplace
-from nullptr.models.jumpgate import Jumpgate
 from nullptr.api import Api
 from .util import *
 from time import sleep, time
@@ -17,10 +13,8 @@ class CommandError(Exception):
   pass
   
 class Commander(CommandLine):
-  def __init__(self, store_dir='data'):
+  def __init__(self, store_file='data/store.npt'):
-    self.store_dir = store_dir
+    self.store = Store(store_file)
-    self.store = Store(store_dir)
-    self.store.load()
     self.agent = self.select_agent()
     self.api = Api(self.store, self.agent)
     self.atlas_builder = AtlasBuilder(self.store, self.api)
@@ -58,6 +52,10 @@ class Commander(CommandLine):
     if agent is None:
       symbol = input('agent name: ')
       agent = self.store.get(Agent, symbol, create=True)
+      api = Api(self.store, agent)
+      faction = input('faction: ')
+      api.register(faction.upper().strip())
+      self.store.flush()
     return agent
   
   def resolve(self, typ, arg):
@@ -107,7 +105,7 @@ class Commander(CommandLine):
     
   def do_cmine(self):
     if not self.has_ship(): return 
-    site = self.ship.location_str
+    site = self.ship.location
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -118,7 +116,7 @@ class Commander(CommandLine):
     self.centcom.set_mission_param(self.ship, 'site', site)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
+    self.centcom.set_mission_param(self.ship, 'contract', contract)
     self.print_mission()
   
   def do_chaul(self):
@@ -140,10 +138,10 @@ class Commander(CommandLine):
     _, m, _, _ = m[0]
     site = self.store.get(Waypoint, m.symbol)
     self.centcom.init_mission(self.ship, 'haul')
-    self.centcom.set_mission_param(self.ship, 'site', site.symbol)
+    self.centcom.set_mission_param(self.ship, 'site', site)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
+    self.centcom.set_mission_param(self.ship, 'contract', contract)
     self.print_mission()
   
   def do_cprobe(self):
@@ -160,15 +158,14 @@ class Commander(CommandLine):
       return
     markets = [ mkt[1] for mkt in m]
     markets = self.analyzer.solve_tsp(markets)
-    hops = ','.join([m.symbol for m in markets])
     self.centcom.init_mission(self.ship, 'probe')
-    self.centcom.set_mission_param(self.ship, 'hops', hops)
+    self.centcom.set_mission_param(self.ship, 'hops', markets)
     self.print_mission()
   
   def do_travel(self, dest):
     dest = self.resolve('Waypoint', dest)
     self.centcom.init_mission(self.ship, 'travel')
-    self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
+    self.centcom.set_mission_param(self.ship, 'dest', dest)
     self.print_mission()
     
   def do_register(self, faction):
@@ -194,7 +191,7 @@ class Commander(CommandLine):
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return
-      system = self.ship.location().system()
+      system = self.ship.location.system
     else:
       system = self.store.get(System, system_str)
     r = self.store.all_members(system, 'Waypoint')
@@ -220,7 +217,7 @@ class Commander(CommandLine):
   def do_jumps(self, waypoint_str=None):
     if waypoint_str is None:
       if not self.has_ship(): return
-      waypoint = self.ship.location()
+      waypoint = self.ship.location
     else:
       waypoint = self.store.get(Waypoint, waypoint_str.upper())
     r = self.api.jumps(waypoint)
@@ -228,7 +225,7 @@ class Commander(CommandLine):
   
   def do_query(self, resource):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     resource = resource.upper()
     print('Found markets:')
     for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@@ -261,7 +258,7 @@ class Commander(CommandLine):
   
   def do_deliver(self):
     if not self.has_ship(): return 
-    site = self.ship.location_str
+    site = self.ship.location
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -290,7 +287,7 @@ class Commander(CommandLine):
   
   def do_go(self, arg):
     if not self.has_ship(): return
-    system = self.ship.location().system()
+    system = self.ship.location.system
     symbol = f'{system}-{arg}'
     dest = self.resolve('Waypoint', symbol)
     self.api.navigate(self.ship, dest)
@@ -324,7 +321,7 @@ class Commander(CommandLine):
   def do_market(self, arg=''):
     if arg == '':
       if not self.has_ship(): return
-      waypoint = self.ship.location()
+      waypoint = self.ship.location
     else:
       waypoint = self.resolve('Waypoint', arg)
     r =  self.api.marketplace(waypoint)
@@ -354,7 +351,7 @@ class Commander(CommandLine):
   
   def do_shipyard(self):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     data = self.api.shipyard(location)
     for s in must_get(data, 'ships'):
       print(s['type'], s['purchasePrice'])
@@ -362,7 +359,7 @@ class Commander(CommandLine):
   def do_jump(self, system_str):
     if not self.has_ship(): return
     if '-' not in system_str:
-      sector = self.ship.location_str.split('-')[0]
+      sector = self.ship.location.system.sector.symbol
       system_str = f'{sector}-{system_str}'
     system = self.resolve('System', system_str)
     self.api.jump(self.ship, system)
@@ -370,7 +367,7 @@ class Commander(CommandLine):
     
   def do_purchase(self, ship_type):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     ship_type = ship_type.upper()
     if not ship_type.startswith('SHIP'):
       ship_type = 'SHIP_' + ship_type

nullptr/models/__init__.py

@@ -0,0 +1,12 @@
+from nullptr.models.base import Base
+from nullptr.models.waypoint import Waypoint
+from nullptr.models.sector import Sector
+from nullptr.models.system import System
+from nullptr.models.agent import Agent
+from nullptr.models.marketplace import Marketplace
+from nullptr.models.jumpgate import Jumpgate
+from nullptr.models.ship import Ship
+from nullptr.models.contract import Contract
+from nullptr.models.survey import Survey
+
+__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

nullptr/models/base.py

@@ -1,18 +1,38 @@
 from copy import deepcopy
 from nullptr.util import sg
 
+class Reference:
+  def __init__(self, typ, symbol, store):
+    self.typ = typ
+    self.symbol = symbol
+    self.store = store
+    
+  @classmethod
+  def create(cls, obj):
+    o = cls(type(obj), obj.symbol, obj.store)
+    return o
+    
+  def resolve(self):
+    self.store.get(self.typ, self.symbol)
+    
+  def __repr__(self):
+    return f'*REF*{self.symbol}.{self.typ.ext()}'
+
 class Base:
   identifier = 'symbol'
-  symbol: str
-  store: object
   
   def __init__(self, symbol, store):
     self.disable_dirty = True
+    self.file_offset = 0
     self.store = store
     self.symbol = symbol
     self.define()
     self.disable_dirty = False
     
+  @classmethod
+  def ext(cls):
+    raise NotImplementedError('no ext')
+    
   def define(self):
     pass
     
@@ -38,10 +58,18 @@ class Base:
       setattr(self, attr, lst)
     
   def __setattr__(self, name, value):
-    if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
+    if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
       self.store.dirty(self)
+    if issubclass(type(value), Base):
+      value = Reference.create(value)
     super().__setattr__(name, value)
   
+  def __getattribute__(self, nm):
+    val = super().__getattribute__(nm)
+    if type(val) == Reference:
+      val = val.resolve()
+    return val
+  
   def update(self, d):
     pass
   
@@ -53,21 +81,14 @@ class Base:
     self.__dict__.update(d)
     self.disable_dirty = False
   
-  def dict(self):
+  def __getstate__(self):
     r = {}
     for k,v in self.__dict__.items():
-      if k in ['store']:
+      if k in ['store','file_offset', 'disable_dirty', 'file_offset']:
         continue
       r[k] = deepcopy(v)
     return r
     
-  def path(self):
-    raise NotImplementedError('path')
-  
-  @classmethod
-  def ext(self):
-    raise NotImplementedError('extension')
-  
   def type(self):
     return self.__class__.__name__
   

nullptr/models/jumpgate.py

@@ -1,7 +1,7 @@
-from .system_member import SystemMember
+from .base import Base
 from dataclasses import field
 
-class Jumpgate(SystemMember):
+class Jumpgate(Base):
   def define(self):
     self.range: int = 0
     self.faction: str = ''

nullptr/models/marketplace.py

@@ -1,10 +1,10 @@
 
-from .system_member import SystemMember
+from .base import Base
 from time import time
 from nullptr.util import *
 from dataclasses import field
 
-class Marketplace(SystemMember):
+class Marketplace(Base):
   def define(self):
     self.imports:list = []
     self.exports:list = []

nullptr/models/survey.py

@@ -1,10 +1,10 @@
 from time import time
 from nullptr.util import *
-from .system_member import SystemMember
+from .base import Base
 
 size_names = ['SMALL','MODERATE','LARGE']
 
-class Survey(SystemMember):
+class Survey(Base):
   identifier = 'signature'
   def define(self):
     self.type: str = ''

nullptr/models/system_member.py

@@ -1,10 +0,0 @@
-from .base import Base
-
-class SystemMember(Base):
-  @classmethod
-  def ext(cls):
-    return 'obj'
-    
-  def system(self):
-    p = self.symbol.split('-')
-    return f'{p[0]}-{p[1]}'

nullptr/models/waypoint.py

@@ -1,8 +1,8 @@
-from .system_member import SystemMember
+from .base import Base
 from nullptr.util import *
 from dataclasses import field
 
-class Waypoint(SystemMember):
+class Waypoint(Base):
   def define(self):
     self.x:int = 0
     self.y:int = 0

nullptr/store.py

@@ -1,30 +1,55 @@
-from nullptr.models.base import Base
+from nullptr.models import *
-from nullptr.models.waypoint import Waypoint
-from nullptr.models.sector import Sector
-from nullptr.models.system import System
-from nullptr.models.agent import Agent
-from nullptr.models.marketplace import Marketplace
-from nullptr.models.system_member import SystemMember
-from nullptr.models.jumpgate import Jumpgate
-from nullptr.models.ship import Ship
-from nullptr.models.contract import Contract
-from nullptr.models.survey import Survey
 from os.path import isfile, dirname, isdir
 import os
 from os.path import basename
 import json
 from .util import *
 from time import time
+import pickle
+from struct import unpack, pack
+
+class ChunkHeader:
+  def __init__(self):
+    self.in_use = True
+    self.size = 0
+    self.used = 0
+    
+  @classmethod
+  def parse(cls, fil):
+    d = fil.read(16)
+    if len(d) < 16:
+      return None
+    #  print(d)
+    o = cls()
+    d, o.used = unpack('<QQ', d)
+    o.size =   d & 0x7fffffffffffffff
+    o.in_use = d & 0x8000000000000000 != 0
+    # print(o)
+    return o
+  
+  def write(self, f):
+    d = self.size
+    if self.in_use:
+      d |= 1 << 63
+    d = pack('<QQ', d, self.used)
+    f.write(d)
+  
+  def __repr__(self):
+    return f'chunk {self.in_use} {self.size} {self.used}'
     
 class Store:
-  def __init__(self, data_dir):
+  def __init__(self, data_file):
     self.init_models()
-    self.data_dir = data_dir
+    self.fil = open_file(data_file)
     self.data = {m: {} for m in self.models}
     self.system_members = {}
     self.dirty_objects = set()
     self.cleanup_interval = 600
     self.last_cleanup = 0
+    self.slack = 0.1
+    self.slack_min = 64
+    self.slack_max = 1024
+    self.load()
   
   def init_models(self):
     self.models = all_subclasses(Base)
@@ -34,53 +59,88 @@ class Store:
   def dirty(self, obj):
     self.dirty_objects.add(obj)
   
-  def path(self, obj):
+  def dump_object(self, obj):
-    return os.path.join(self.data_dir, obj.path())
+    return pickle.dumps(obj)
   
-  def load_file(self, path):
+  def load_object(self, data, offset):
-    if not isfile(path):
+    obj = pickle.loads(data)
-      return None
+    obj.file_offset = offset
-    fn = basename(path)
+    obj.disable_dirty = False
-    ext = fn.split('.')[-1]
+    self.hold(obj)
-    symbol = fn.split('.')[0]
-    if ext not in self.extensions:
-      return None
-    with open(path) as f:
-      data = json.load(f)
-    
-    typ = self.extensions[ext]
-    obj = self.create(typ, symbol)
-    obj.load(data)
-    obj.store = self
-    return obj
     
   def load(self):
     cnt = 0
     start_time = time()
-    for fil in list_files(self.data_dir, True):
+    
-      self.load_file(fil)
+    self.fil.seek(0)
+    offset = 0
+    while (hdr := ChunkHeader.parse(self.fil)):
+      if not hdr.in_use: continue
+      data = self.fil.read(hdr.used)
+      self.load_object(data, offset)
+      self.fil.seek(hdr.size - hdr.used, 1)
+      offset = self.fil.tell()
       cnt += 1
+      
     dur = time() - start_time
     print(f'loaded {cnt} objects in {dur:.2f} seconds')
   
+  def allocate_chunk(self, sz):
+    used = sz
+    slack = sz * self.slack
+    slack = min(slack, self.slack_max)
+    slack = max(slack, self.slack_min)
+    sz += slack
+    self.fil.seek(0, 2)
+    offset = self.fil.tell()
+    h = ChunkHeader()
+    h.size = sz
+    h.used = used
+    h.write(self.fil)
+    return offset, h
+    
   def store(self, obj):
-    path = self.path(obj)
+    data = self.dump_object(obj)
-    path_dir = dirname(path)
+    osize = len(data)
-    data = obj.dict()
+    # is there an existing chunk for this obj?
-    if not isdir(path_dir):
+    if obj.file_offset > 0:
-      os.makedirs(path_dir, exist_ok=True)
+      # read chunk hdr
-    with open(path, 'w') as f:
+      self.fil.seek(obj.file_offset)
-      json.dump(data, f, indent=2)
+      hdr = ChunkHeader.parse(self.fil)
+      csize = hdr.size
+      # if the chunk is too small
+      if csize < osize:
+        # free the chunk
+        hdr.in_use = False
+        # force a new chunk
+        obj.file_offset = 0
+      else:
+        # if it is big enough, update the used field
+        hdr.used = osize
+      self.fil.seek(obj.file_offset)
+      hdr.write(self.fil)
         
-  def create(self, typ, symbol):
+    if obj.file_offset == 0:
-    obj = typ(symbol, self)
+      obj.file_offset, hdr = self.allocate_chunk(osize)
+    self.fil.write(data)
+    slack = b'\x00' * (hdr.size - hdr.used)
+    self.fil.write(slack)
+    
+  def hold(self, obj):
+    typ = type(obj)
+    symbol = obj.symbol
+    obj.store = self
     self.data[typ][symbol] = obj
-    if issubclass(typ, SystemMember):
+    if hasattr(typ, 'system'):
-      system_str = obj.system()
+      system_str = obj.system.symbol
-      
       if system_str not in self.system_members:
         self.system_members[system_str] = set()
       self.system_members[system_str].add(obj)
+  
+  def create(self, typ, symbol):
+    obj = typ(symbol, self)
+    self.hold(obj)
+    self.dirty(obj)
     return obj
     
   def get(self, typ, symbol, create=False):
@@ -122,7 +182,7 @@ class Store:
     if type(system) == System:
       system = system.symbol
   
-    if system not in self.system_members:
+    if 'system' not in self.system_members:
       return 
       print('typ', typ)
     for m in self.system_members[system]:
@@ -139,9 +199,9 @@ class Store:
         if o.is_expired():
           expired.append(o)
     for o in expired:
-      path = o.path()
+      
-      if isfile(path):
+      # TODO
-        os.remove(path)
+      
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
     # print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@@ -153,6 +213,7 @@ class Store:
     for obj in self.dirty_objects:
       it += 1
       self.store(obj)
+    self.fil.flush()
     self.dirty_objects = set()
     dur = time() - start_time
     #  print(f'flush done {it} items {dur:.2f}')

nullptr/util.py

@@ -1,20 +1,15 @@
 from datetime import datetime
 from math import ceil
 import os
-from os.path import isfile
+from os.path import isfile, dirname
 
-def list_files(path, recursive=False):
+def open_file(fn):
-  if recursive:
+  d = dirname(fn)
-   for p, dirnames, fils in os.walk(path):
+  os.makedirs(d, exist_ok=True)
-     for f in fils:
+  if isfile(fn):
-       fil = os.path.join(p, f)
+    return open(fn, 'rb+')
-       yield fil
   else:
-    for f in os.listdir(path):
+    return open(fn, 'ab+')
-      fil = os.path.join(path, f)
-      if not isfile(fil):
-        continue
-      yield fil
 
 def must_get(d, k):
   if type(k) == str:

store.md

@@ -3,6 +3,10 @@ This project uses a custom database format just because we can.
 
 The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
 
+The store disk format is optimized for two things:
+* Loading the entire contents in memory
+* Writing changed objects back to disk
+
 ## Objects
 First lets discuss what we are storing.
 
@@ -30,21 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict
 * store.load(fil) loads all objects
 * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
 * store.all(type) generator for all objects of a goven type
+* store.delete(typ, symbol)
 * store.cleanup() removes all expired objects
 * store.flush() writes all dirty objects to disk
-* 
+* store.defrag() consolidates the store file to minimize storage and loading time
 
 type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
 
 # file format
 Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
 
-the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header:
+The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
 
-* Magic
+A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.
-* Blocksize in bytes
-* Number of blocks
-* Root file
-* Free file
 
-A block is prefixed by a pointer to the next block of that file. In the last block of the file, the pointer is 0. So if you have 1000 byte blocks, each block takes 1008 bytes of space.