Dockerfile

@@ -9,4 +9,4 @@ ADD --chown=user . /app
 RUN chmod +x /app/main.py
 VOLUME /data
 ENTRYPOINT [ "python3", "/app/main.py"]
-CMD ["-s", "/data/store.npt"]
+CMD ["-s", "/data/"]

main.py

@@ -1,9 +1,8 @@
-#!/usr/bin/env python3
 import argparse
 from nullptr.commander import Commander
-from nullptr.models.base import Base
+
 def main(args):
-  c = Commander(args.store_file)
+  c = Commander(args.store_dir)
   c.run()
   
 # X1-AG74-41076A
@@ -11,6 +10,6 @@ def main(args):
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
-  parser.add_argument('-s', '--store-file', default='data/store.npt')
+  parser.add_argument('-s', '--store-dir', default='data')
   args = parser.parse_args()
   main(args)

nullptr/analyzer.py

@@ -44,16 +44,16 @@ class Analyzer:
       location = self.store.get(Waypoint, location)
     mkts = self.find_markets(resource, sellbuy)
     candidates = []
-    origin = location.system
+    origin = self.store.get(System, location.system())
     for typ, m in mkts:
-      system = m.waypoint.system
+      system = self.store.get(System, m.system())
       d = origin.distance(system)
       candidates.append((typ, m, d))
     possibles = sorted(candidates, key=lambda m: m[2])
     possibles = possibles[:10]
     results = []
     for typ,m,d in possibles:
-      system = m.waypoint.system
+      system = self.store.get(System, m.system())
       p = self.find_path(origin, system)
       if p is None: continue
       results.append((typ,m,d,len(p)))
@@ -83,7 +83,9 @@ class Analyzer:
       for s in jg.systems:
         if s in seen:  continue
         seen.add(s)
-        dest.add(SearchNode(s, o))
+        system = self.store.get(System, s)
+        if system is None: continue
+        dest.add(SearchNode(system, o))
     if len(dest) == 0: 
       return None
     return self.find_path(dest, to, depth-1, seen)

nullptr/api.py

@@ -69,7 +69,6 @@ class Api:
     }
     result = self.request('post', 'register', data, need_token=False)
     token = mg(result, 'token')
-    self.agent.update(mg(result, 'agent'))
     self.agent.token = token
   
   def info(self):
@@ -80,10 +79,7 @@ class Api:
   def list_systems(self, page=1):
     data = self.request('get', 'systems', params={'page': page})
     #pprint(self.last_meta)
-    systems = self.store.update_list(System, data)
-    for s in data:
-      self.store.update_list(Waypoint, mg(s, 'waypoints'))
-    return systems
+    return self.store.update_list(System, data)
   
   def list_waypoints(self, system):
     data = self.request('get', f'systems/{system}/waypoints/')
@@ -91,12 +87,13 @@ class Api:
     return self.store.update_list(Waypoint, data)
   
   def marketplace(self, waypoint):
-    system = waypoint.system
+    system = waypoint.system()
+    symbol = str(waypoint)
     data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
     return self.store.update(Marketplace, data)
   
   def jumps(self, waypoint):
-    data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
+    data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
     symbol = str(waypoint)
     return self.store.update(Jumpgate, data, symbol)
   

nullptr/atlas_builder.py

@@ -31,12 +31,12 @@ class AtlasBuilder:
       if 'UNCHARTED' in w.traits:
         continue
       if 'MARKETPLACE' in w.traits:
-        print(f'marketplace at {w}')
         self.api.marketplace(w)
+        print(f'marketplace at {w}')
         sleep(0.5)
       if w.type == 'JUMP_GATE':
-        print(f'jumpgate at {w}')
         self.api.jumps(w)
+        print(f'jumpgate at {w}')
         
   def all_waypoints(self, systems):
     for s in systems:

nullptr/commander.py

@@ -2,7 +2,11 @@ from nullptr.command_line import CommandLine
 from nullptr.store import Store
 from nullptr.analyzer import  Analyzer
 import argparse
-from nullptr.models import *
+from nullptr.models.agent import Agent
+from nullptr.models.system import System
+from nullptr.models.waypoint import Waypoint
+from nullptr.models.marketplace import Marketplace
+from nullptr.models.jumpgate import Jumpgate
 from nullptr.api import Api
 from .util import *
 from time import sleep, time
@@ -13,8 +17,10 @@ class CommandError(Exception):
   pass
   
 class Commander(CommandLine):
-  def __init__(self, store_file='data/store.npt'):
-    self.store = Store(store_file)
+  def __init__(self, store_dir='data'):
+    self.store_dir = store_dir
+    self.store = Store(store_dir)
+    self.store.load()
     self.agent = self.select_agent()
     self.api = Api(self.store, self.agent)
     self.atlas_builder = AtlasBuilder(self.store, self.api)
@@ -50,23 +56,8 @@ class Commander(CommandLine):
     agents = self.store.all(Agent)
     agent = next(agents, None)
     if agent is None:
-      agent = self.agent_setup()
-    return agent
-  
-  def agent_setup(self):
-    symbol = input('agent name: ')
-    agent = self.store.get(Agent, symbol, create=True)
-    api = Api(self.store, agent)
-    self.api = api
-    faction = input('faction: ')
-    api.register(faction.upper().strip())
-    print('=== agent:')
-    print(agent)
-    print('=== ships')
-    self.do_ships('r')
-    print('=== contracts')
-    self.do_contracts('r')
-    self.store.flush()
+      symbol = input('agent name: ')
+      agent = self.store.get(Agent, symbol, create=True)
     return agent
   
   def resolve(self, typ, arg):
@@ -116,7 +107,7 @@ class Commander(CommandLine):
     
   def do_cmine(self):
     if not self.has_ship(): return 
-    site = self.ship.location
+    site = self.ship.location_str
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -127,12 +118,12 @@ class Commander(CommandLine):
     self.centcom.set_mission_param(self.ship, 'site', site)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract)
+    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
     self.print_mission()
   
   def do_chaul(self):
     if not self.has_ship(): return     
-    if len(self.ship.cargo) > 0:
+    if len(ship.cargo) > 0:
       raise CommandError('please dump cargo first')
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
@@ -149,10 +140,10 @@ class Commander(CommandLine):
     _, m, _, _ = m[0]
     site = self.store.get(Waypoint, m.symbol)
     self.centcom.init_mission(self.ship, 'haul')
-    self.centcom.set_mission_param(self.ship, 'site', site)
+    self.centcom.set_mission_param(self.ship, 'site', site.symbol)
     self.centcom.set_mission_param(self.ship, 'resource', resource)
     self.centcom.set_mission_param(self.ship, 'dest', destination)
-    self.centcom.set_mission_param(self.ship, 'contract', contract)
+    self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
     self.print_mission()
   
   def do_cprobe(self):
@@ -169,14 +160,15 @@ class Commander(CommandLine):
       return
     markets = [ mkt[1] for mkt in m]
     markets = self.analyzer.solve_tsp(markets)
+    hops = ','.join([m.symbol for m in markets])
     self.centcom.init_mission(self.ship, 'probe')
-    self.centcom.set_mission_param(self.ship, 'hops', markets)
+    self.centcom.set_mission_param(self.ship, 'hops', hops)
     self.print_mission()
   
   def do_travel(self, dest):
     dest = self.resolve('Waypoint', dest)
     self.centcom.init_mission(self.ship, 'travel')
-    self.centcom.set_mission_param(self.ship, 'dest', dest)
+    self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
     self.print_mission()
     
   def do_register(self, faction):
@@ -202,10 +194,9 @@ class Commander(CommandLine):
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return
-      system = self.ship.location.system
+      system = self.ship.location().system()
     else:
       system = self.store.get(System, system_str)
-    print(f'=== waypoints in {system}')
     r = self.store.all_members(system, 'Waypoint')
     for w in r:
       traits = []
@@ -229,7 +220,7 @@ class Commander(CommandLine):
   def do_jumps(self, waypoint_str=None):
     if waypoint_str is None:
       if not self.has_ship(): return
-      waypoint = self.ship.location
+      waypoint = self.ship.location()
     else:
       waypoint = self.store.get(Waypoint, waypoint_str.upper())
     r = self.api.jumps(waypoint)
@@ -237,7 +228,7 @@ class Commander(CommandLine):
   
   def do_query(self, resource):
     if not self.has_ship(): return
-    location = self.ship.location
+    location = self.ship.location()
     resource = resource.upper()
     print('Found markets:')
     for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@@ -270,7 +261,7 @@ class Commander(CommandLine):
   
   def do_deliver(self):
     if not self.has_ship(): return 
-    site = self.ship.location
+    site = self.ship.location_str
     contract = self.active_contract()
     delivery = contract.unfinished_delivery()
     if delivery is None:
@@ -299,7 +290,7 @@ class Commander(CommandLine):
   
   def do_go(self, arg):
     if not self.has_ship(): return
-    system = self.ship.location.system
+    system = self.ship.location().system()
     symbol = f'{system}-{arg}'
     dest = self.resolve('Waypoint', symbol)
     self.api.navigate(self.ship, dest)
@@ -333,7 +324,7 @@ class Commander(CommandLine):
   def do_market(self, arg=''):
     if arg == '':
       if not self.has_ship(): return
-      waypoint = self.ship.location
+      waypoint = self.ship.location()
     else:
       waypoint = self.resolve('Waypoint', arg)
     r =  self.api.marketplace(waypoint)
@@ -363,7 +354,7 @@ class Commander(CommandLine):
   
   def do_shipyard(self):
     if not self.has_ship(): return
-    location = self.ship.location
+    location = self.ship.location()
     data = self.api.shipyard(location)
     for s in must_get(data, 'ships'):
       print(s['type'], s['purchasePrice'])
@@ -371,7 +362,7 @@ class Commander(CommandLine):
   def do_jump(self, system_str):
     if not self.has_ship(): return
     if '-' not in system_str:
-      sector = self.ship.location.system.sector.symbol
+      sector = self.ship.location_str.split('-')[0]
       system_str = f'{sector}-{system_str}'
     system = self.resolve('System', system_str)
     self.api.jump(self.ship, system)
@@ -379,7 +370,7 @@ class Commander(CommandLine):
     
   def do_purchase(self, ship_type):
     if not self.has_ship(): return
-    location = self.ship.location
+    location = self.ship.location()
     ship_type = ship_type.upper()
     if not ship_type.startswith('SHIP'):
       ship_type = 'SHIP_' + ship_type

nullptr/missions/base.py

@@ -26,14 +26,9 @@ class MissionParam:
     elif self.cls == int:
       return int(val)
     elif self.cls == list:
-      if type(val) == str:
-        return [i.strip() for i in val.split(',')]
-      return val
+      return [i.strip() for i in val.split(',')]
     elif issubclass(self.cls, Base):
-      if type(val) == str:
-        data = store.get(self.cls, val)
-      else:
-        data = val
+      data = store.get(self.cls, val)
       if data is None:
         raise ValueError('object not found')
       return data.symbol
@@ -184,13 +179,14 @@ class BaseMission(Mission):
     traject = self.st('traject')
     if traject is None or traject == []:
       return 'done'
-    dest = traject[-1]
-    loc = self.ship.location
+    dest = self.store.get(Waypoint, traject[-1])
+    loc = self.ship.location()
+    print(dest, loc)
     if dest == loc:
       self.sts('traject', None)
       return 'done'
     hop = traject.pop(0)
-    if type(hop) == Waypoint:
+    if len(hop.split('-')) == 3:
       self.api.navigate(self.ship, hop)
       self.next_step = self.ship.arrival
     else:
@@ -204,23 +200,22 @@ class BaseMission(Mission):
   def step_calculate_traject(self, dest):
     if type(dest) == str:
       dest = self.store.get(Waypoint, dest)
-    loc = self.ship.location
-    loc_sys = loc.system
+    loc = self.ship.location()
+    loc_sys = self.store.get(System, loc.system())
     loc_jg = self.analyzer.get_jumpgate(loc_sys)
-    loc_jg_wp = self.store.get(Waypoint, loc_jg.symbol)
-    dest_sys = dest.system
+    dest_sys = self.store.get(System, dest.system())
     dest_jg = self.analyzer.get_jumpgate(dest_sys)
     if dest_sys == loc_sys:
-      result = [dest]
+      result = [dest.symbol]
       self.sts('traject', result)
       return
     path = self.analyzer.find_path(loc_sys, dest_sys)
     result = []
     if loc.symbol != loc_jg.symbol:
-      result.append(loc_jg_wp)
-    result += [s for s in path[1:]]
+      result.append(loc_jg.symbol)
+    result += [s.symbol for s in path[1:]]
     if dest_jg.symbol != dest.symbol:
-      result.append(dest)
+      result.append(dest.symbol)
     self.sts('traject', result)
     print(result)
     return result

nullptr/models/__init__.py

@@ -1,12 +0,0 @@
-from nullptr.models.base import Base
-from nullptr.models.waypoint import Waypoint
-from nullptr.models.sector import Sector
-from nullptr.models.system import System
-from nullptr.models.agent import Agent
-from nullptr.models.marketplace import Marketplace
-from nullptr.models.jumpgate import Jumpgate
-from nullptr.models.ship import Ship
-from nullptr.models.contract import Contract
-from nullptr.models.survey import Survey
-
-__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

nullptr/models/agent.py

@@ -8,6 +8,9 @@ class Agent(Base):
   def update(self, d):
     self.seta('credits', d)
     
+  def path(self):
+    return f'{self.symbol}.{self.ext()}'
+  
   @classmethod
   def ext(self):
     return 'agt'

nullptr/models/base.py

@@ -1,38 +1,18 @@
 from copy import deepcopy
 from nullptr.util import sg
 
-class Reference:
-  def __init__(self, typ, symbol, store):
-    self.typ = typ
-    self.symbol = symbol
-    self.store = store
-  
-  @classmethod
-  def create(cls, obj):
-    o = cls(type(obj), obj.symbol, obj.store)
-    return o
-    
-  def resolve(self):
-    return self.store.get(self.typ, self.symbol)
-    
-  def __repr__(self):
-    return f'*REF*{self.symbol}.{self.typ.ext()}'
-
 class Base:
   identifier = 'symbol'
+  symbol: str
+  store: object
   
   def __init__(self, symbol, store):
     self.disable_dirty = True
-    self.file_offset = None
     self.store = store
     self.symbol = symbol
     self.define()
     self.disable_dirty = False
     
-  @classmethod
-  def ext(cls):
-    raise NotImplementedError('no ext')
-    
   def define(self):
     pass
     
@@ -40,13 +20,7 @@ class Base:
     return hash((str(type(self)), self.symbol))
   
   def __eq__(self, other):
-    return type(self) == type(other) and self.symbol == other.symbol
-  
-  def get_system(self):
-    parts = self.symbol.split('-')
-    system_str = f'{parts[0]}-{parts[1]}'
-    system = self.store.get('System', system_str, create=True)
-    return system
+    return self.symbol == other.symbol and type(self) == type(other)
     
   def seta(self, attr,  d, name=None, interp=None):
     if name is None:
@@ -57,30 +31,17 @@ class Base:
         val = interp(val)
       setattr(self, attr, val)
   
-  def setlst(self, attr, d, name, member, interp=None):
+  def setlst(self, attr, d, name, member):
     val = sg(d, name)
     if val is not None:
-      lst = []
-      for x in val:
-        val = sg(x, member)
-        if interp is not None:
-          val = interp(val)
-        lst.append(val)
+      lst = [sg(x, member) for x in val]
       setattr(self, attr, lst)
     
   def __setattr__(self, name, value):
-    if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
+    if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
       self.store.dirty(self)
-    if issubclass(type(value), Base):
-      value = Reference.create(value)
     super().__setattr__(name, value)
   
-  def __getattribute__(self, nm):
-    val = super().__getattribute__(nm)
-    if type(val) == Reference:
-      val = val.resolve()
-    return val
-  
   def update(self, d):
     pass
   
@@ -92,15 +53,27 @@ class Base:
     self.__dict__.update(d)
     self.disable_dirty = False
     
+  def dict(self):
+    r = {}
+    for k,v in self.__dict__.items():
+      if k in ['store']:
+        continue
+      r[k] = deepcopy(v)
+    return r
+    
+  def path(self):
+    raise NotImplementedError('path')
+  
+  @classmethod
+  def ext(self):
+    raise NotImplementedError('extension')
+  
   def type(self):
     return self.__class__.__name__
   
   def __str__(self):
     return self.f()
     
-  def __repr__(self):
-    return self.f()
-    
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:

nullptr/models/contract.py

@@ -18,6 +18,9 @@ class Contract(Base):
   def ext(cls):
     return 'cnt'
   
+  def path(self):
+    return f'contracts/{self.symbol}.{self.ext()}'
+  
   def is_expired(self):
     return time() > self.expires
   

nullptr/models/jumpgate.py

@@ -1,17 +1,14 @@
-from .base import Base
-from .system import System
+from .system_member import SystemMember
 from dataclasses import field
 
-class Jumpgate(Base):
+class Jumpgate(SystemMember):
   def define(self):
     self.range: int = 0
     self.faction: str = ''
     self.systems: list = []
-    self.system = self.get_system()
   
   def update(self, d):
-    getter = self.store.getter(System, create=True)
-    self.setlst('systems', d, 'connectedSystems', 'symbol', interp=getter)
+    self.setlst('systems', d, 'connectedSystems', 'symbol')
     self.seta('faction', d, 'factionSymbol')
     self.seta('range', d, 'jumpRange')
     
@@ -19,9 +16,13 @@ class Jumpgate(Base):
   def ext(self):
     return 'jmp'
   
+  def path(self):
+    sector, system, _ = self.symbol.split('-')
+    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
+  
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:
       r += '\n'
-      r += '\n'.join([s.symbol for s in self.systems])
+      r += '\n'.join(self.systems)
     return r

nullptr/models/marketplace.py

@@ -1,23 +1,16 @@
 
-from .base import Base
+from .system_member import SystemMember
 from time import time
 from nullptr.util import *
 from dataclasses import field
-from nullptr.models import Waypoint
 
-class Marketplace(Base):
+class Marketplace(SystemMember):
   def define(self):
     self.imports:list = []
     self.exports:list = []
     self.exchange:list = []
     self.prices:dict = {}
     self.last_prices:int = 0
-    self.set_waypoint()
-    self.system = self.get_system()
-  
-  def set_waypoint(self):
-    waypoint = self.store.get(Waypoint, self.symbol, create=True)
-    self.waypoint = waypoint
   
   def update(self, d):
     self.setlst('imports', d, 'imports', 'symbol')
@@ -52,6 +45,10 @@ class Marketplace(Base):
       return 'X'
     return '?'
     
+  def path(self):
+    sector, system, _ = self.symbol.split('-')
+    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
+  
   def f(self, detail=1):
     r = self.symbol
     if detail > 1:

nullptr/models/ship.py

@@ -1,7 +1,7 @@
 from .base import Base
 from time import time
 from nullptr.util import *
-from nullptr.models import Waypoint
+from dataclasses import dataclass, field
 
 class Ship(Base):  
   def define(self):
@@ -10,7 +10,7 @@ class Ship(Base):
     self.status:str = ''
     self.cargo_capacity:int = 0
     self.cargo_units:int = 0
-    self.location = None
+    self.location_str = ''
     self.cooldown:int = 0
     self.arrival:int = 0
     self.fuel_current:int = 0
@@ -22,10 +22,16 @@ class Ship(Base):
   def ext(self):
     return 'shp'
   
+  def location(self):
+    return self.store.get('Waypoint', self.location_str)
+    
+  def path(self):
+    agent = self.symbol.split('-')[0]
+    return f'{agent}/{self.symbol}.{self.ext()}'
+
   def update(self, d):
     self.seta('status', d, 'nav.status')
-    getter = self.store.getter(Waypoint, create=True)
-    self.seta('location', d, 'nav.waypointSymbol', interp=getter)
+    self.seta('location_str', d, 'nav.waypointSymbol')
     self.seta('cargo_capacity', d, 'cargo.capacity')
     self.seta('cargo_units', d, 'cargo.units')
     self.seta('fuel_capacity', d, 'fuel.capacity')
@@ -89,7 +95,7 @@ class Ship(Base):
     if detail > 1:
       r += ' ' + self.status
       r += f' [{self.fuel_current}/{self.fuel_capacity}]'
-      r += ' ' + str(self.location)
+      r += ' ' + str(self.location())
       if self.is_travelling():
         r += f' [A: {arrival}]'
       if self.is_cooldown():

nullptr/models/survey.py

@@ -1,10 +1,10 @@
 from time import time
 from nullptr.util import *
-from .base import Base
+from .system_member import SystemMember
 
 size_names = ['SMALL','MODERATE','LARGE']
 
-class Survey(Base):
+class Survey(SystemMember):
   identifier = 'signature'
   def define(self):
     self.type: str = ''
@@ -18,6 +18,11 @@ class Survey(Base):
   def ext(cls):
     return 'svy'
   
+  def path(self):
+    sector, system, waypoint, signature = self.symbol.split('-')
+    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
+    
+  
   def is_expired(self):
     return time() > self.expires or self.exhausted
   

nullptr/models/system.py

@@ -17,6 +17,10 @@ class System(Base):
   def ext(self):
     return 'stm'
   
+  def path(self):
+    sector, symbol = self.symbol.split('-')
+    return f'atlas/{sector}/{symbol[0:1]}/{self.symbol}.{self.ext()}'
+  
   def distance(self, other):
     return int(sqrt((self.x - other.x) ** 2 + (self.y - other.y) ** 2))
   

nullptr/models/system_member.py

@@ -0,0 +1,10 @@
+from .base import Base
+
+class SystemMember(Base):
+  @classmethod
+  def ext(cls):
+    return 'obj'
+    
+  def system(self):
+    p = self.symbol.split('-')
+    return f'{p[0]}-{p[1]}'

nullptr/models/waypoint.py

@@ -1,15 +1,14 @@
-from .base import Base, Reference
-from nullptr.models.system import System
+from .system_member import SystemMember
 from nullptr.util import *
+from dataclasses import field
 
-class Waypoint(Base):
+class Waypoint(SystemMember):
   def define(self):
     self.x:int = 0
     self.y:int = 0
     self.type:str = 'unknown'
     self.traits:list = []
     self.faction:str = ''
-    self.system = self.get_system()
   
   def update(self, d):
     self.seta('x', d)
@@ -22,3 +21,6 @@ class Waypoint(Base):
   def ext(self):
     return 'way'
     
+  def path(self):
+    sector, system, _ = self.symbol.split('-')
+    return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'

nullptr/store.py

@@ -1,74 +1,30 @@
-from nullptr.models import *
+from nullptr.models.base import Base
+from nullptr.models.waypoint import Waypoint
+from nullptr.models.sector import Sector
+from nullptr.models.system import System
+from nullptr.models.agent import Agent
+from nullptr.models.marketplace import Marketplace
+from nullptr.models.system_member import SystemMember
+from nullptr.models.jumpgate import Jumpgate
+from nullptr.models.ship import Ship
+from nullptr.models.contract import Contract
+from nullptr.models.survey import Survey
 from os.path import isfile, dirname, isdir
 import os
 from os.path import basename
 import json
 from .util import *
 from time import time
-import pickle
-from struct import unpack, pack
-from functools import partial
-from io import BytesIO
-
-class StorePickler(pickle.Pickler):
-  def persistent_id(self, obj):
-    return "STORE" if type(obj) == Store else None
-
-class StoreUnpickler(pickle.Unpickler):
-  def __init__(self, stream, store):
-    self.store = store
-    super().__init__(stream)
-    
-  def persistent_load(self, pers_id):
-    if pers_id == "STORE":
-      return self.store
-    raise pickle.UnpicklingError("I don know the persid!")
-     
-        
-class ChunkHeader:
-  def __init__(self):
-    self.offset = 0
-    self.in_use = True
-    self.size = 0
-    self.used = 0
-    
-  @classmethod
-  def parse(cls, fil):
-    offset = fil.tell()
-    d = fil.read(16)
-    if len(d) < 16:
-      return None
-    o = cls()
-    o.offset = offset
-    d, o.used = unpack('<QQ', d)
-    o.size =   d & 0x7fffffffffffffff
-    o.in_use = d & 0x8000000000000000 != 0
-    # print(o)
-    return o
-  
-  def write(self, f):
-    d = self.size
-    if self.in_use:
-      d |= 1 << 63
-    d = pack('<QQ', d, self.used)
-    f.write(d)
-  
-  def __repr__(self):
-    return f'chunk {self.in_use} {self.size} {self.used}'
 
 class Store:
-  def __init__(self, data_file):
+  def __init__(self, data_dir):
     self.init_models()
-    self.fil = open_file(data_file)
+    self.data_dir = data_dir
     self.data = {m: {} for m in self.models}
     self.system_members = {}
     self.dirty_objects = set()
     self.cleanup_interval = 600
     self.last_cleanup = 0
-    self.slack = 0.1
-    self.slack_min = 64
-    self.slack_max = 1024
-    self.load()
   
   def init_models(self):
     self.models = all_subclasses(Base)
@@ -78,98 +34,53 @@ class Store:
   def dirty(self, obj):
     self.dirty_objects.add(obj)
     
-  def dump_object(self, obj):
-    buf = BytesIO()
-    p = StorePickler(buf)
-    p.dump(obj)
-    return buf.getvalue()
+  def path(self, obj):
+    return os.path.join(self.data_dir, obj.path())
     
-  def load_object(self, data, offset):
-    buf = BytesIO(data)
-    p = StoreUnpickler(buf, self)
-    obj = p.load()
-    obj.file_offset = offset
-    obj.disable_dirty = False
-    self.hold(obj)
+  def load_file(self, path):
+    if not isfile(path):
+      return None
+    fn = basename(path)
+    ext = fn.split('.')[-1]
+    symbol = fn.split('.')[0]
+    if ext not in self.extensions:
+      return None
+    with open(path) as f:
+      data = json.load(f)
+    
+    typ = self.extensions[ext]
+    obj = self.create(typ, symbol)
+    obj.load(data)
+    obj.store = self
+    return obj
   
   def load(self):
     cnt = 0
     start_time = time()
-    
-    self.fil.seek(0)
-    offset = 0
-    while (hdr := ChunkHeader.parse(self.fil)):
-      # print(hdr)
-      if not hdr.in_use: 
-        self.fil.seek(hdr.size, 1)
-        continue
-      data = self.fil.read(hdr.used)
-      self.load_object(data, offset)
-      self.fil.seek(hdr.size - hdr.used, 1)
-      offset = self.fil.tell()
+    for fil in list_files(self.data_dir, True):
+      self.load_file(fil)
       cnt += 1
-      
     dur = time() - start_time
     print(f'loaded {cnt} objects in {dur:.2f} seconds')
   
-  def allocate_chunk(self, sz):
-    used = sz
-    slack = sz * self.slack
-    slack = min(slack, self.slack_max)
-    slack = max(slack, self.slack_min)
-    sz += int(slack)
-    self.fil.seek(0, 2)
-    offset = self.fil.tell()
-    h = ChunkHeader()
-    h.size = sz
-    h.used = used
-    h.offset = self.fil.tell()
-    h.write(self.fil)
-    return offset, h
-    
   def store(self, obj):
-    data = self.dump_object(obj)
-    osize = len(data)
-    # is there an existing chunk for this obj?
-    if obj.file_offset is not None:
-      # read chunk hdr
-      self.fil.seek(obj.file_offset)
-      hdr = ChunkHeader.parse(self.fil)
-      csize = hdr.size
-      # if the chunk is too small
-      if csize < osize:
-        # free the chunk
-        hdr.in_use = False
-        # force a new chunk
-        obj.file_offset = None
-      else:
-        # if it is big enough, update the used field
-        hdr.used = osize
-      self.fil.seek(hdr.offset)
-      hdr.write(self.fil)
-        
-    if obj.file_offset is None:
-      obj.file_offset, hdr = self.allocate_chunk(osize)
-    # print(type(obj).__name__, hdr)
-    self.fil.write(data)
-    slack = b'\x00' * (hdr.size - hdr.used)
-    self.fil.write(slack)
-    
-  def hold(self, obj):
-    typ = type(obj)
-    symbol = obj.symbol
-    obj.store = self
-    self.data[typ][symbol] = obj
-    if hasattr(obj, 'system') and obj.system != None:
-      system_str = obj.system.symbol
-      if system_str not in self.system_members:
-        self.system_members[system_str] = set()
-      self.system_members[system_str].add(obj)
+    path = self.path(obj)
+    path_dir = dirname(path)
+    data = obj.dict()
+    if not isdir(path_dir):
+      os.makedirs(path_dir, exist_ok=True)
+    with open(path, 'w') as f:
+      json.dump(data, f, indent=2)
   
   def create(self, typ, symbol):
     obj = typ(symbol, self)
-    self.hold(obj)
-    self.dirty(obj)
+    self.data[typ][symbol] = obj
+    if issubclass(typ, SystemMember):
+      system_str = obj.system()
+      
+      if system_str not in self.system_members:
+        self.system_members[system_str] = set()
+      self.system_members[system_str].add(obj)
     return obj
     
   def get(self, typ, symbol, create=False):
@@ -185,11 +96,6 @@ class Store:
         return None
     return self.data[typ][symbol]
   
-  def getter(self, typ, create=False):
-    if type(typ) == str and typ in self.model_names:
-      typ = self.model_names[typ]
-    return partial(self.get, typ, create=create)
-  
   def update(self, typ, data, symbol=None):
     if type(typ) == str and typ in self.model_names:
       typ = self.model_names[typ] 
@@ -218,7 +124,7 @@ class Store:
   
     if system not in self.system_members:
       return 
-    
+      print('typ', typ)
     for m in self.system_members[system]:
       if typ is None or type(m) == typ:
         yield m
@@ -233,9 +139,9 @@ class Store:
         if o.is_expired():
           expired.append(o)
     for o in expired:
-      
-      # TODO
-      
+      path = o.path()
+      if isfile(path):
+        os.remove(path)
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
     # print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@@ -247,7 +153,6 @@ class Store:
     for obj in self.dirty_objects:
       it += 1
       self.store(obj)
-    self.fil.flush()
     self.dirty_objects = set()
     dur = time() - start_time
     #  print(f'flush done {it} items {dur:.2f}')

nullptr/util.py

@@ -1,15 +1,20 @@
 from datetime import datetime
 from math import ceil
 import os
-from os.path import isfile, dirname
+from os.path import isfile
 
-def open_file(fn):
-  d = dirname(fn)
-  os.makedirs(d, exist_ok=True)
-  if isfile(fn):
-    return open(fn, 'rb+')
+def list_files(path, recursive=False):
+  if recursive:
+   for p, dirnames, fils in os.walk(path):
+     for f in fils:
+       fil = os.path.join(p, f)
+       yield fil
   else:
-    return open(fn, 'ab+')
+    for f in os.listdir(path):
+      fil = os.path.join(path, f)
+      if not isfile(fil):
+        continue
+      yield fil
       
 def must_get(d, k):
   if type(k) == str:
@@ -53,7 +58,7 @@ def pretty(d, ident=0, detail=2):
     return d.f(detail)
   r = ''
   idt = ' ' * ident
-  if type(d) in [list, set]:
+  if type(d) == list:
     r += 'lst'
     for i in d:
       r += '\n' + idt + pretty(i, ident + 1, detail)

store.md

@@ -3,10 +3,6 @@ This project uses a custom database format just because we can.
 
 The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
 
-The store disk format is optimized for two things:
-* Loading the entire contents in memory
-* Writing changed objects back to disk
-
 ## Objects
 First lets discuss what we are storing.
 
@@ -34,17 +30,12 @@ An index is a dict with a string as key and a list of objects as value. The dict
 * store.load(fil) loads all objects
 * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
 * store.all(type) generator for all objects of a goven type
-* store.delete(typ, symbol)
 * store.cleanup() removes all expired objects
 * store.flush() writes all dirty objects to disk
-* store.defrag() consolidates the store file to minimize storage and loading time
+* 
 
 type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
 
 # file format
-Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
-
-The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
-
-A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.
+the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header.