Update base.py

Dockerfile

@@ -9,4 +9,4 @@ ADD --chown=user . /app
 RUN chmod +x /app/main.py
 VOLUME /data
 ENTRYPOINT [ "python3", "/app/main.py"]
-CMD ["-s", "/data/"]
+CMD ["-s", "/data/store.npt"]

main.py

@@ -1,8 +1,9 @@
+#!/usr/bin/env python3
 import argparse
 from nullptr.commander import Commander
-
+from nullptr.models.base import Base
 def main(args):
-  c = Commander(args.store_dir)
+  c = Commander(args.store_file)
   c.run()
   
 # X1-AG74-41076A
@@ -10,6 +11,6 @@ def main(args):
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--store-dir', default='data')
+  parser.add_argument('-s', '--store-file', default='data/store.npt')
   args = parser.parse_args()
   main(args)

nullptr/analyzer.py

@@ -44,16 +44,16 @@ class Analyzer:
       location = self.store.get(Waypoint, location)
     mkts = self.find_markets(resource, sellbuy)
     candidates = []
-    origin = self.store.get(System, location.system())
+    origin = location.system
     for typ, m in mkts:
-      system = self.store.get(System, m.system())
+      system = m.waypoint.system
       d = origin.distance(system)
       candidates.append((typ, m, d))
     possibles = sorted(candidates, key=lambda m: m[2])
     possibles = possibles[:10]
     results = []
     for typ,m,d in possibles:
-      system = self.store.get(System, m.system())
+      system = m.waypoint.system
       p = self.find_path(origin, system)
       if p is None: continue
       results.append((typ,m,d,len(p)))
@@ -83,9 +83,7 @@ class Analyzer:
       for s in jg.systems:
         if s in seen:  continue
         seen.add(s)
-        system = self.store.get(System, s)
-        if system is None: continue
-        dest.add(SearchNode(system, o))
+        dest.add(SearchNode(s, o))
     if len(dest) == 0: 
       return None
     return self.find_path(dest, to, depth-1, seen)

nullptr/api.py

@@ -69,6 +69,7 @@ class Api:
     }
     result = self.request('post', 'register', data, need_token=False)
     token = mg(result, 'token')
+    self.agent.update(mg(result, 'agent'))
     self.agent.token = token
   
   def info(self):
@@ -79,7 +80,10 @@ class Api:
   def list_systems(self, page=1):
     data = self.request('get', 'systems', params={'page': page})
     #pprint(self.last_meta)
-    return self.store.update_list(System, data)
+    systems = self.store.update_list(System, data)
+    for s in data:
+      self.store.update_list(Waypoint, mg(s, 'waypoints'))
+    return systems
   
   def list_waypoints(self, system):
     data = self.request('get', f'systems/{system}/waypoints/')
@@ -87,13 +91,12 @@ class Api:
     return self.store.update_list(Waypoint, data)
   
   def marketplace(self, waypoint):
-    system = waypoint.system()
-    symbol = str(waypoint)
+    system = waypoint.system
     data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
     return self.store.update(Marketplace, data)
   
   def jumps(self, waypoint):
-    data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
+    data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
     symbol = str(waypoint)
     return self.store.update(Jumpgate, data, symbol)
   

nullptr/atlas_builder.py

@@ -31,12 +31,12 @@ class AtlasBuilder:
       if 'UNCHARTED' in w.traits:
         continue
       if 'MARKETPLACE' in w.traits:
-        self.api.marketplace(w)
         print(f'marketplace at {w}')
+        self.api.marketplace(w)
         sleep(0.5)
       if w.type == 'JUMP_GATE':
-        self.api.jumps(w)
         print(f'jumpgate at {w}')
+        self.api.jumps(w)
         
   def all_waypoints(self, systems):
     for s in systems:

nullptr/commander.py

@@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine
 from nullptr.store import Store
 from nullptr.analyzer import  Analyzer
 import argparse
-from nullptr.models.agent import Agent
-from nullptr.models.system import System
-from nullptr.models.waypoint import Waypoint
-from nullptr.models.marketplace import Marketplace
-from nullptr.models.jumpgate import Jumpgate
+from nullptr.models import *
 from nullptr.api import Api
 from .util import *
 from time import sleep, time
@@ -17,10 +13,8 @@ class CommandError(Exception):
   pass
   
 class Commander(CommandLine):
-  def __init__(self, store_dir='data'):
-    self.store_dir = store_dir
-    self.store = Store(store_dir)
-    self.store.load()
+  def __init__(self, store_file='data/store.npt'):
+    self.store = Store(store_file)
     self.agent = self.select_agent()
     self.api = Api(self.store, self.agent)
     self.atlas_builder = AtlasBuilder(self.store, self.api)
@@ -56,8 +50,23 @@ class Commander(CommandLine):
     agents = self.store.all(Agent)
     agent = next(agents, None)
     if agent is None:
-      symbol = input('agent name: ')
-      agent = self.store.get(Agent, symbol, create=True)
+      agent = self.agent_setup()
+    return agent
+  
+  def agent_setup(self):
+    symbol = input('agent name: ')
+    agent = self.store.get(Agent, symbol, create=True)
+    api = Api(self.store, agent)
+    self.api = api
+    faction = input('faction: ')
+    api.register(faction.upper().strip())
+    print('=== agent:')
+    print(agent)
+    print('=== ships')
+    self.do_ships('r')
+    print('=== contracts')
+    self.do_contracts('r')
+    self.store.flush()
     return agent
   
   def resolve(self, typ, arg):
@@ -107,7 +116,7 @@ class Commander(CommandLine):
     
   def do_cmine(self):
     if not self.has_ship(): return 
-    site = self.ship.location_str
+    site = self.ship.location
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -118,12 +127,12 @@ class Commander(CommandLine):
     self.centcom.set_mission_param(self.ship, 'site', site)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
+    self.centcom.set_mission_param(self.ship, 'contract', contract)
     self.print_mission()
   
   def do_chaul(self):
     if not self.has_ship(): return     
-    if len(ship.cargo) > 0:
+    if len(self.ship.cargo) > 0:
       raise CommandError('please dump cargo first')
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
@@ -140,10 +149,10 @@ class Commander(CommandLine):
     _, m, _, _ = m[0]
     site = self.store.get(Waypoint, m.symbol)
     self.centcom.init_mission(self.ship, 'haul')
-    self.centcom.set_mission_param(self.ship, 'site', site.symbol)
+    self.centcom.set_mission_param(self.ship, 'site', site)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
+    self.centcom.set_mission_param(self.ship, 'contract', contract)
     self.print_mission()
   
   def do_cprobe(self):
@@ -160,15 +169,14 @@ class Commander(CommandLine):
       return
     markets = [ mkt[1] for mkt in m]
     markets = self.analyzer.solve_tsp(markets)
-    hops = ','.join([m.symbol for m in markets])
     self.centcom.init_mission(self.ship, 'probe')
-    self.centcom.set_mission_param(self.ship, 'hops', hops)
+    self.centcom.set_mission_param(self.ship, 'hops', markets)
     self.print_mission()
   
   def do_travel(self, dest):
     dest = self.resolve('Waypoint', dest)
     self.centcom.init_mission(self.ship, 'travel')
-    self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
+    self.centcom.set_mission_param(self.ship, 'dest', dest)
     self.print_mission()
     
   def do_register(self, faction):
@@ -194,9 +202,10 @@ class Commander(CommandLine):
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return
-      system = self.ship.location().system()
+      system = self.ship.location.system
     else:
       system = self.store.get(System, system_str)
+    print(f'=== waypoints in {system}')
     r = self.store.all_members(system, 'Waypoint')
     for w in r:
       traits = []
@@ -220,7 +229,7 @@ class Commander(CommandLine):
   def do_jumps(self, waypoint_str=None):
     if waypoint_str is None:
       if not self.has_ship(): return
-      waypoint = self.ship.location()
+      waypoint = self.ship.location
     else:
       waypoint = self.store.get(Waypoint, waypoint_str.upper())
     r = self.api.jumps(waypoint)
@@ -228,7 +237,7 @@ class Commander(CommandLine):
   
   def do_query(self, resource):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     resource = resource.upper()
     print('Found markets:')
     for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@@ -261,7 +270,7 @@ class Commander(CommandLine):
   
   def do_deliver(self):
     if not self.has_ship(): return 
-    site = self.ship.location_str
+    site = self.ship.location
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -290,7 +299,7 @@ class Commander(CommandLine):
   
   def do_go(self, arg):
     if not self.has_ship(): return
-    system = self.ship.location().system()
+    system = self.ship.location.system
     symbol = f'{system}-{arg}'
     dest = self.resolve('Waypoint', symbol)
     self.api.navigate(self.ship, dest)
@@ -324,7 +333,7 @@ class Commander(CommandLine):
   def do_market(self, arg=''):
     if arg == '':
       if not self.has_ship(): return
-      waypoint = self.ship.location()
+      waypoint = self.ship.location
     else:
       waypoint = self.resolve('Waypoint', arg)
     r =  self.api.marketplace(waypoint)
@@ -354,7 +363,7 @@ class Commander(CommandLine):
   
   def do_shipyard(self):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     data = self.api.shipyard(location)
     for s in must_get(data, 'ships'):
       print(s['type'], s['purchasePrice'])
@@ -362,7 +371,7 @@ class Commander(CommandLine):
   def do_jump(self, system_str):
     if not self.has_ship(): return
     if '-' not in system_str:
-      sector = self.ship.location_str.split('-')[0]
+      sector = self.ship.location.system.sector.symbol
       system_str = f'{sector}-{system_str}'
     system = self.resolve('System', system_str)
     self.api.jump(self.ship, system)
@@ -370,7 +379,7 @@ class Commander(CommandLine):
     
   def do_purchase(self, ship_type):
     if not self.has_ship(): return
-    location = self.ship.location()
+    location = self.ship.location
     ship_type = ship_type.upper()
     if not ship_type.startswith('SHIP'):
       ship_type = 'SHIP_' + ship_type

nullptr/missions/base.py

@@ -26,9 +26,14 @@ class MissionParam:
     elif self.cls == int:
       return int(val)
     elif self.cls == list:
-      return [i.strip() for i in val.split(',')]
+      if type(val) == str:
+        return [i.strip() for i in val.split(',')]
+      return val
     elif issubclass(self.cls, Base):
-      data = store.get(self.cls, val)
+      if type(val) == str:
+        data = store.get(self.cls, val)
+      else:
+        data = val
       if data is None:
         raise ValueError('object not found')
       return data.symbol
@@ -179,14 +184,13 @@ class BaseMission(Mission):
     traject = self.st('traject')
     if traject is None or traject == []:
       return 'done'
-    dest = self.store.get(Waypoint, traject[-1])
-    loc = self.ship.location()
-    print(dest, loc)
+    dest = traject[-1]
+    loc = self.ship.location
     if dest == loc:
       self.sts('traject', None)
       return 'done'
     hop = traject.pop(0)
-    if len(hop.split('-')) == 3:
+    if type(hop) == Waypoint:
       self.api.navigate(self.ship, hop)
       self.next_step = self.ship.arrival
     else:
@@ -200,22 +204,23 @@ class BaseMission(Mission):
   def step_calculate_traject(self, dest):
     if type(dest) == str:
       dest = self.store.get(Waypoint, dest)
-    loc = self.ship.location()
-    loc_sys = self.store.get(System, loc.system())
+    loc = self.ship.location
+    loc_sys = loc.system
     loc_jg = self.analyzer.get_jumpgate(loc_sys)
-    dest_sys = self.store.get(System, dest.system())
+    loc_jg_wp = self.store.get(Waypoint, loc_jg.symbol)
+    dest_sys = dest.system
     dest_jg = self.analyzer.get_jumpgate(dest_sys)
     if dest_sys == loc_sys:
-      result = [dest.symbol]
+      result = [dest]
       self.sts('traject', result)
       return
     path = self.analyzer.find_path(loc_sys, dest_sys)
     result = []
     if loc.symbol != loc_jg.symbol:
-      result.append(loc_jg.symbol)
-    result += [s.symbol for s in path[1:]]
+      result.append(loc_jg_wp)
+    result += [s for s in path[1:]]
     if dest_jg.symbol != dest.symbol:
-      result.append(dest.symbol)
+      result.append(dest)
     self.sts('traject', result)
     print(result)
     return result

nullptr/models/__init__.py

@@ -0,0 +1,12 @@
+from nullptr.models.base import Base
+from nullptr.models.waypoint import Waypoint
+from nullptr.models.sector import Sector
+from nullptr.models.system import System
+from nullptr.models.agent import Agent
+from nullptr.models.marketplace import Marketplace
+from nullptr.models.jumpgate import Jumpgate
+from nullptr.models.ship import Ship
+from nullptr.models.contract import Contract
+from nullptr.models.survey import Survey
+
+__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

nullptr/models/agent.py

@@ -8,9 +8,6 @@ class Agent(Base):
   def update(self, d):
     self.seta('credits', d)
     
-  def path(self):
-    return f'{self.symbol}.{self.ext()}'
-  
   @classmethod
   def ext(self):
     return 'agt'

nullptr/models/base.py

@@ -1,18 +1,38 @@
 from copy import deepcopy
 from nullptr.util import sg
 
+class Reference:
+  def __init__(self, typ, symbol, store):
+    self.typ = typ
+    self.symbol = symbol
+    self.store = store
+  
+  @classmethod
+  def create(cls, obj):
+    o = cls(type(obj), obj.symbol, obj.store)
+    return o
+    
+  def resolve(self):
+    return self.store.get(self.typ, self.symbol)
+    
+  def __repr__(self):
+    return f'*REF*{self.symbol}.{self.typ.ext()}'
+
 class Base:
   identifier = 'symbol'
-  symbol: str
-  store: object
   
   def __init__(self, symbol, store):
     self.disable_dirty = True
+    self.file_offset = None
     self.store = store
     self.symbol = symbol
     self.define()
     self.disable_dirty = False
     
+  @classmethod
+  def ext(cls):
+    raise NotImplementedError('no ext')
+    
   def define(self):
     pass
     
@@ -20,7 +40,13 @@ class Base:
     return hash((str(type(self)), self.symbol))
   
   def __eq__(self, other):
-    return self.symbol == other.symbol and type(self) == type(other)
+    return type(self) == type(other) and self.symbol == other.symbol
+  
+  def get_system(self):
+    parts = self.symbol.split('-')
+    system_str = f'{parts[0]}-{parts[1]}'
+    system = self.store.get('System', system_str, create=True)
+    return system
     
   def seta(self, attr,  d, name=None, interp=None):
     if name is None:
@@ -31,17 +57,30 @@ class Base:
         val = interp(val)
       setattr(self, attr, val)
   
-  def setlst(self, attr, d, name, member):
+  def setlst(self, attr, d, name, member, interp=None):
     val = sg(d, name)
     if val is not None:
-      lst = [sg(x, member) for x in val]
+      lst = []
+      for x in val:
+        val = sg(x, member)
+        if interp is not None:
+          val = interp(val)
+        lst.append(val)
       setattr(self, attr, lst)
     
   def __setattr__(self, name, value):
-    if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
+    if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
       self.store.dirty(self)
+    if issubclass(type(value), Base):
+      value = Reference.create(value)
     super().__setattr__(name, value)
   
+  def __getattribute__(self, nm):
+    val = super().__getattribute__(nm)
+    if type(val) == Reference:
+      val = val.resolve()
+    return val
+  
   def update(self, d):
     pass
   
@@ -53,27 +92,15 @@ class Base:
     self.__dict__.update(d)
     self.disable_dirty = False
     
-  def dict(self):
-    r = {}
-    for k,v in self.__dict__.items():
-      if k in ['store']:
-        continue
-      r[k] = deepcopy(v)
-    return r
-    
-  def path(self):
-    raise NotImplementedError('path')
-  
-  @classmethod
-  def ext(self):
-    raise NotImplementedError('extension')
-  
   def type(self):
     return self.__class__.__name__
   
   def __str__(self):
     return self.f()
     
+  def __repr__(self):
+    return self.f()
+    
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:

nullptr/models/contract.py

@@ -18,9 +18,6 @@ class Contract(Base):
   def ext(cls):
     return 'cnt'
   
-  def path(self):
-    return f'contracts/{self.symbol}.{self.ext()}'
-  
   def is_expired(self):
     return time() > self.expires
   

nullptr/models/jumpgate.py

@@ -1,14 +1,17 @@
-from .system_member import SystemMember
+from .base import Base
+from .system import System
 from dataclasses import field
 
-class Jumpgate(SystemMember):
+class Jumpgate(Base):
   def define(self):
     self.range: int = 0
     self.faction: str = ''
     self.systems: list = []
+    self.system = self.get_system()
   
   def update(self, d):
-    self.setlst('systems', d, 'connectedSystems', 'symbol')
+    getter = self.store.getter(System, create=True)
+    self.setlst('systems', d, 'connectedSystems', 'symbol', interp=getter)
     self.seta('faction', d, 'factionSymbol')
     self.seta('range', d, 'jumpRange')
     
@@ -16,13 +19,9 @@ class Jumpgate(SystemMember):
   def ext(self):
     return 'jmp'
   
-  def path(self):
-    sector, system, _ = self.symbol.split('-')
-    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
-  
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:
       r += '\n'
-      r += '\n'.join(self.systems)
+      r += '\n'.join([s.symbol for s in self.systems])
     return r

nullptr/models/marketplace.py

@@ -1,16 +1,23 @@
 
-from .system_member import SystemMember
+from .base import Base
 from time import time
 from nullptr.util import *
 from dataclasses import field
+from nullptr.models import Waypoint
 
-class Marketplace(SystemMember):
+class Marketplace(Base):
   def define(self):
     self.imports:list = []
     self.exports:list = []
     self.exchange:list = []
     self.prices:dict = {}
     self.last_prices:int = 0
+    self.set_waypoint()
+    self.system = self.get_system()
+  
+  def set_waypoint(self):
+    waypoint = self.store.get(Waypoint, self.symbol, create=True)
+    self.waypoint = waypoint
   
   def update(self, d):
     self.setlst('imports', d, 'imports', 'symbol')
@@ -45,10 +52,6 @@ class Marketplace(SystemMember):
       return 'X'
     return '?'
   
-  def path(self):
-    sector, system, _ = self.symbol.split('-')
-    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
-  
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:

nullptr/models/ship.py

@@ -1,7 +1,7 @@
 from .base import Base
 from time import time
 from nullptr.util import *
-from dataclasses import dataclass, field
+from nullptr.models import Waypoint
 
 class Ship(Base):  
   def define(self):
@@ -10,7 +10,7 @@ class Ship(Base):
     self.status:str = ''
     self.cargo_capacity:int = 0
     self.cargo_units:int = 0
-    self.location_str = ''
+    self.location = None
     self.cooldown:int = 0
     self.arrival:int = 0
     self.fuel_current:int = 0
@@ -22,16 +22,10 @@ class Ship(Base):
   def ext(self):
     return 'shp'
   
-  def location(self):
-    return self.store.get('Waypoint', self.location_str)
-    
-  def path(self):
-    agent = self.symbol.split('-')[0]
-    return f'{agent}/{self.symbol}.{self.ext()}'
-
   def update(self, d):
     self.seta('status', d, 'nav.status')
-    self.seta('location_str', d, 'nav.waypointSymbol')
+    getter = self.store.getter(Waypoint, create=True)
+    self.seta('location', d, 'nav.waypointSymbol', interp=getter)
     self.seta('cargo_capacity', d, 'cargo.capacity')
     self.seta('cargo_units', d, 'cargo.units')
     self.seta('fuel_capacity', d, 'fuel.capacity')
@@ -95,7 +89,7 @@ class Ship(Base):
     if detail > 1:
       r += ' ' + self.status
       r += f' [{self.fuel_current}/{self.fuel_capacity}]'
-      r += ' ' + str(self.location())
+      r += ' ' + str(self.location)
       if self.is_travelling():
         r += f' [A: {arrival}]'
       if self.is_cooldown():

nullptr/models/survey.py

@@ -1,10 +1,10 @@
 from time import time
 from nullptr.util import *
-from .system_member import SystemMember
+from .base import Base
 
 size_names = ['SMALL','MODERATE','LARGE']
 
-class Survey(SystemMember):
+class Survey(Base):
   identifier = 'signature'
   def define(self):
     self.type: str = ''
@@ -18,11 +18,6 @@ class Survey(SystemMember):
   def ext(cls):
     return 'svy'
 
-  def path(self):
-    sector, system, waypoint, signature = self.symbol.split('-')
-    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
-    
-  
   def is_expired(self):
     return time() > self.expires or self.exhausted
   

nullptr/models/system.py

@@ -17,10 +17,6 @@ class System(Base):
   def ext(self):
     return 'stm'
   
-  def path(self):
-    sector, symbol = self.symbol.split('-')
-    return f'atlas/{sector}/{symbol[0:1]}/{self.symbol}.{self.ext()}'
-  
   def distance(self, other):
     return int(sqrt((self.x - other.x) ** 2 + (self.y - other.y) ** 2))
   

nullptr/models/system_member.py

@@ -1,10 +0,0 @@
-from .base import Base
-
-class SystemMember(Base):
-  @classmethod
-  def ext(cls):
-    return 'obj'
-    
-  def system(self):
-    p = self.symbol.split('-')
-    return f'{p[0]}-{p[1]}'

nullptr/models/waypoint.py

@@ -1,14 +1,15 @@
-from .system_member import SystemMember
+from .base import Base, Reference
+from nullptr.models.system import System
 from nullptr.util import *
-from dataclasses import field
 
-class Waypoint(SystemMember):
+class Waypoint(Base):
   def define(self):
     self.x:int = 0
     self.y:int = 0
     self.type:str = 'unknown'
     self.traits:list = []
     self.faction:str = ''
+    self.system = self.get_system()
     
   def update(self, d):
     self.seta('x', d)
@@ -21,6 +22,3 @@ class Waypoint(SystemMember):
   def ext(self):
     return 'way'
     
-  def path(self):
-    sector, system, _ = self.symbol.split('-')
-    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'

nullptr/store.py

@@ -1,30 +1,74 @@
-from nullptr.models.base import Base
-from nullptr.models.waypoint import Waypoint
-from nullptr.models.sector import Sector
-from nullptr.models.system import System
-from nullptr.models.agent import Agent
-from nullptr.models.marketplace import Marketplace
-from nullptr.models.system_member import SystemMember
-from nullptr.models.jumpgate import Jumpgate
-from nullptr.models.ship import Ship
-from nullptr.models.contract import Contract
-from nullptr.models.survey import Survey
+from nullptr.models import *
 from os.path import isfile, dirname, isdir
 import os
 from os.path import basename
 import json
 from .util import *
 from time import time
+import pickle
+from struct import unpack, pack
+from functools import partial
+from io import BytesIO
+
+class StorePickler(pickle.Pickler):
+  def persistent_id(self, obj):
+    return "STORE" if type(obj) == Store else None
+
+class StoreUnpickler(pickle.Unpickler):
+  def __init__(self, stream, store):
+    self.store = store
+    super().__init__(stream)
+    
+  def persistent_load(self, pers_id):
+    if pers_id == "STORE":
+      return self.store
+    raise pickle.UnpicklingError("I don know the persid!")
+     
+        
+class ChunkHeader:
+  def __init__(self):
+    self.offset = 0
+    self.in_use = True
+    self.size = 0
+    self.used = 0
+    
+  @classmethod
+  def parse(cls, fil):
+    offset = fil.tell()
+    d = fil.read(16)
+    if len(d) < 16:
+      return None
+    o = cls()
+    o.offset = offset
+    d, o.used = unpack('<QQ', d)
+    o.size =   d & 0x7fffffffffffffff
+    o.in_use = d & 0x8000000000000000 != 0
+    # print(o)
+    return o
+  
+  def write(self, f):
+    d = self.size
+    if self.in_use:
+      d |= 1 << 63
+    d = pack('<QQ', d, self.used)
+    f.write(d)
+  
+  def __repr__(self):
+    return f'chunk {self.in_use} {self.size} {self.used}'
     
 class Store:
-  def __init__(self, data_dir):
+  def __init__(self, data_file):
     self.init_models()
-    self.data_dir = data_dir
+    self.fil = open_file(data_file)
     self.data = {m: {} for m in self.models}
     self.system_members = {}
     self.dirty_objects = set()
     self.cleanup_interval = 600
     self.last_cleanup = 0
+    self.slack = 0.1
+    self.slack_min = 64
+    self.slack_max = 1024
+    self.load()
   
   def init_models(self):
     self.models = all_subclasses(Base)
@@ -34,53 +78,98 @@ class Store:
   def dirty(self, obj):
     self.dirty_objects.add(obj)
   
-  def path(self, obj):
-    return os.path.join(self.data_dir, obj.path())
+  def dump_object(self, obj):
+    buf = BytesIO()
+    p = StorePickler(buf)
+    p.dump(obj)
+    return buf.getvalue()
   
-  def load_file(self, path):
-    if not isfile(path):
-      return None
-    fn = basename(path)
-    ext = fn.split('.')[-1]
-    symbol = fn.split('.')[0]
-    if ext not in self.extensions:
-      return None
-    with open(path) as f:
-      data = json.load(f)
-    
-    typ = self.extensions[ext]
-    obj = self.create(typ, symbol)
-    obj.load(data)
-    obj.store = self
-    return obj
+  def load_object(self, data, offset):
+    buf = BytesIO(data)
+    p = StoreUnpickler(buf, self)
+    obj = p.load()
+    obj.file_offset = offset
+    obj.disable_dirty = False
+    self.hold(obj)
     
   def load(self):
     cnt = 0
     start_time = time()
-    for fil in list_files(self.data_dir, True):
-      self.load_file(fil)
+    
+    self.fil.seek(0)
+    offset = 0
+    while (hdr := ChunkHeader.parse(self.fil)):
+      # print(hdr)
+      if not hdr.in_use: 
+        self.fil.seek(hdr.size, 1)
+        continue
+      data = self.fil.read(hdr.used)
+      self.load_object(data, offset)
+      self.fil.seek(hdr.size - hdr.used, 1)
+      offset = self.fil.tell()
       cnt += 1
+      
     dur = time() - start_time
     print(f'loaded {cnt} objects in {dur:.2f} seconds')
   
+  def allocate_chunk(self, sz):
+    used = sz
+    slack = sz * self.slack
+    slack = min(slack, self.slack_max)
+    slack = max(slack, self.slack_min)
+    sz += int(slack)
+    self.fil.seek(0, 2)
+    offset = self.fil.tell()
+    h = ChunkHeader()
+    h.size = sz
+    h.used = used
+    h.offset = self.fil.tell()
+    h.write(self.fil)
+    return offset, h
+    
   def store(self, obj):
-    path = self.path(obj)
-    path_dir = dirname(path)
-    data = obj.dict()
-    if not isdir(path_dir):
-      os.makedirs(path_dir, exist_ok=True)
-    with open(path, 'w') as f:
-      json.dump(data, f, indent=2)
+    data = self.dump_object(obj)
+    osize = len(data)
+    # is there an existing chunk for this obj?
+    if obj.file_offset is not None:
+      # read chunk hdr
+      self.fil.seek(obj.file_offset)
+      hdr = ChunkHeader.parse(self.fil)
+      csize = hdr.size
+      # if the chunk is too small
+      if csize < osize:
+        # free the chunk
+        hdr.in_use = False
+        # force a new chunk
+        obj.file_offset = None
+      else:
+        # if it is big enough, update the used field
+        hdr.used = osize
+      self.fil.seek(hdr.offset)
+      hdr.write(self.fil)
         
-  def create(self, typ, symbol):
-    obj = typ(symbol, self)
+    if obj.file_offset is None:
+      obj.file_offset, hdr = self.allocate_chunk(osize)
+    # print(type(obj).__name__, hdr)
+    self.fil.write(data)
+    slack = b'\x00' * (hdr.size - hdr.used)
+    self.fil.write(slack)
+    
+  def hold(self, obj):
+    typ = type(obj)
+    symbol = obj.symbol
+    obj.store = self
     self.data[typ][symbol] = obj
-    if issubclass(typ, SystemMember):
-      system_str = obj.system()
-      
+    if hasattr(obj, 'system') and obj.system != None:
+      system_str = obj.system.symbol
       if system_str not in self.system_members:
         self.system_members[system_str] = set()
       self.system_members[system_str].add(obj)
+  
+  def create(self, typ, symbol):
+    obj = typ(symbol, self)
+    self.hold(obj)
+    self.dirty(obj)
     return obj
     
   def get(self, typ, symbol, create=False):
@@ -96,6 +185,11 @@ class Store:
         return None
     return self.data[typ][symbol]
   
+  def getter(self, typ, create=False):
+    if type(typ) == str and typ in self.model_names:
+      typ = self.model_names[typ]
+    return partial(self.get, typ, create=create)
+  
   def update(self, typ, data, symbol=None):
     if type(typ) == str and typ in self.model_names:
       typ = self.model_names[typ] 
@@ -124,7 +218,7 @@ class Store:
     
     if system not in self.system_members:
       return 
-      print('typ', typ)
+    
     for m in self.system_members[system]:
       if typ is None or type(m) == typ:
         yield m
@@ -139,9 +233,9 @@ class Store:
         if o.is_expired():
           expired.append(o)
     for o in expired:
-      path = o.path()
-      if isfile(path):
-        os.remove(path)
+      
+      # TODO
+      
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
     # print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@@ -153,6 +247,7 @@ class Store:
     for obj in self.dirty_objects:
       it += 1
       self.store(obj)
+    self.fil.flush()
     self.dirty_objects = set()
     dur = time() - start_time
     #  print(f'flush done {it} items {dur:.2f}')

nullptr/util.py

@@ -1,20 +1,15 @@
 from datetime import datetime
 from math import ceil
 import os
-from os.path import isfile
+from os.path import isfile, dirname
 
-def list_files(path, recursive=False):
-  if recursive:
-   for p, dirnames, fils in os.walk(path):
-     for f in fils:
-       fil = os.path.join(p, f)
-       yield fil
+def open_file(fn):
+  d = dirname(fn)
+  os.makedirs(d, exist_ok=True)
+  if isfile(fn):
+    return open(fn, 'rb+')
   else:
-    for f in os.listdir(path):
-      fil = os.path.join(path, f)
-      if not isfile(fil):
-        continue
-      yield fil
+    return open(fn, 'ab+')
 
 def must_get(d, k):
   if type(k) == str:
@@ -58,7 +53,7 @@ def pretty(d, ident=0, detail=2):
     return d.f(detail)
   r = ''
   idt = ' ' * ident
-  if type(d) == list:
+  if type(d) in [list, set]:
     r += 'lst'
     for i in d:
       r += '\n' + idt + pretty(i, ident + 1, detail)

store.md

@@ -3,6 +3,10 @@ This project uses a custom database format just because we can.
 
 The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
 
+The store disk format is optimized for two things:
+* Loading the entire contents in memory
+* Writing changed objects back to disk
+
 ## Objects
 First lets discuss what we are storing.
 
@@ -30,12 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict
 * store.load(fil) loads all objects
 * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
 * store.all(type) generator for all objects of a goven type
+* store.delete(typ, symbol)
 * store.cleanup() removes all expired objects
 * store.flush() writes all dirty objects to disk
-* 
+* store.defrag() consolidates the store file to minimize storage and loading time
 
 type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
 
 # file format
-the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header.
+Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
+
+The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
+
+A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.