Compare commits

...

10 Commits

Author SHA1 Message Date
Richard Bronkhorst
537615e582 Update base.py 2023-07-12 22:36:29 +02:00
Richard Bronkhorst
3d3ceeab91 Fly me to the moon! 2023-07-12 22:26:25 +02:00
Richard Bronkhorst
00db50687a Update atlas_builder.py, commander.py and eleven other files 2023-07-11 22:09:57 +02:00
Richard Bronkhorst
97296e1859 Update analyzer.py, commander.py and six other files 2023-07-11 17:48:51 +02:00
Richard Bronkhorst
269b5cf537 Update api.py, base.py and two other files 2023-07-11 07:17:13 +02:00
richmans
ea34bcfab7 . 2023-07-10 21:10:49 +02:00
Richard Bronkhorst
b2f2dc520e Update waypoint.py and store.py 2023-07-10 20:12:29 +02:00
Richard Bronkhorst
b1e3621490 New store setup 2023-07-10 19:25:01 +02:00
richmans
6537db3c03 a 2023-07-03 21:57:53 +02:00
Richard Bronkhorst
0553d9d6cc Update store.md 2023-07-03 21:52:25 +02:00
21 changed files with 331 additions and 208 deletions

View File

@ -9,4 +9,4 @@ ADD --chown=user . /app
RUN chmod +x /app/main.py
VOLUME /data
ENTRYPOINT [ "python3", "/app/main.py"]
CMD ["-s", "/data/"]
CMD ["-s", "/data/store.npt"]

7
main.py Normal file → Executable file
View File

@ -1,8 +1,9 @@
#!/usr/bin/env python3
import argparse
from nullptr.commander import Commander
from nullptr.models.base import Base
def main(args):
c = Commander(args.store_dir)
c = Commander(args.store_file)
c.run()
# X1-AG74-41076A
@ -10,6 +11,6 @@ def main(args):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--store-dir', default='data')
parser.add_argument('-s', '--store-file', default='data/store.npt')
args = parser.parse_args()
main(args)

View File

@ -44,16 +44,16 @@ class Analyzer:
location = self.store.get(Waypoint, location)
mkts = self.find_markets(resource, sellbuy)
candidates = []
origin = self.store.get(System, location.system())
origin = location.system
for typ, m in mkts:
system = self.store.get(System, m.system())
system = m.waypoint.system
d = origin.distance(system)
candidates.append((typ, m, d))
possibles = sorted(candidates, key=lambda m: m[2])
possibles = possibles[:10]
results = []
for typ,m,d in possibles:
system = self.store.get(System, m.system())
system = m.waypoint.system
p = self.find_path(origin, system)
if p is None: continue
results.append((typ,m,d,len(p)))
@ -83,9 +83,7 @@ class Analyzer:
for s in jg.systems:
if s in seen: continue
seen.add(s)
system = self.store.get(System, s)
if system is None: continue
dest.add(SearchNode(system, o))
dest.add(SearchNode(s, o))
if len(dest) == 0:
return None
return self.find_path(dest, to, depth-1, seen)

View File

@ -69,6 +69,7 @@ class Api:
}
result = self.request('post', 'register', data, need_token=False)
token = mg(result, 'token')
self.agent.update(mg(result, 'agent'))
self.agent.token = token
def info(self):
@ -79,7 +80,10 @@ class Api:
def list_systems(self, page=1):
data = self.request('get', 'systems', params={'page': page})
#pprint(self.last_meta)
return self.store.update_list(System, data)
systems = self.store.update_list(System, data)
for s in data:
self.store.update_list(Waypoint, mg(s, 'waypoints'))
return systems
def list_waypoints(self, system):
data = self.request('get', f'systems/{system}/waypoints/')
@ -87,13 +91,12 @@ class Api:
return self.store.update_list(Waypoint, data)
def marketplace(self, waypoint):
system = waypoint.system()
symbol = str(waypoint)
system = waypoint.system
data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
return self.store.update(Marketplace, data)
def jumps(self, waypoint):
data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
symbol = str(waypoint)
return self.store.update(Jumpgate, data, symbol)

View File

@ -31,12 +31,12 @@ class AtlasBuilder:
if 'UNCHARTED' in w.traits:
continue
if 'MARKETPLACE' in w.traits:
self.api.marketplace(w)
print(f'marketplace at {w}')
self.api.marketplace(w)
sleep(0.5)
if w.type == 'JUMP_GATE':
self.api.jumps(w)
print(f'jumpgate at {w}')
self.api.jumps(w)
def all_waypoints(self, systems):
for s in systems:

View File

@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine
from nullptr.store import Store
from nullptr.analyzer import Analyzer
import argparse
from nullptr.models.agent import Agent
from nullptr.models.system import System
from nullptr.models.waypoint import Waypoint
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models import *
from nullptr.api import Api
from .util import *
from time import sleep, time
@ -17,10 +13,8 @@ class CommandError(Exception):
pass
class Commander(CommandLine):
def __init__(self, store_dir='data'):
self.store_dir = store_dir
self.store = Store(store_dir)
self.store.load()
def __init__(self, store_file='data/store.npt'):
self.store = Store(store_file)
self.agent = self.select_agent()
self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api)
@ -56,8 +50,23 @@ class Commander(CommandLine):
agents = self.store.all(Agent)
agent = next(agents, None)
if agent is None:
symbol = input('agent name: ')
agent = self.store.get(Agent, symbol, create=True)
agent = self.agent_setup()
return agent
def agent_setup(self):
symbol = input('agent name: ')
agent = self.store.get(Agent, symbol, create=True)
api = Api(self.store, agent)
self.api = api
faction = input('faction: ')
api.register(faction.upper().strip())
print('=== agent:')
print(agent)
print('=== ships')
self.do_ships('r')
print('=== contracts')
self.do_contracts('r')
self.store.flush()
return agent
def resolve(self, typ, arg):
@ -107,7 +116,7 @@ class Commander(CommandLine):
def do_cmine(self):
if not self.has_ship(): return
site = self.ship.location_str
site = self.ship.location
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@ -118,12 +127,12 @@ class Commander(CommandLine):
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission()
def do_chaul(self):
if not self.has_ship(): return
if len(ship.cargo) > 0:
if len(self.ship.cargo) > 0:
raise CommandError('please dump cargo first')
contract = self.active_contract()
delivery = contract.unfinished_delivery()
@ -140,10 +149,10 @@ class Commander(CommandLine):
_, m, _, _ = m[0]
site = self.store.get(Waypoint, m.symbol)
self.centcom.init_mission(self.ship, 'haul')
self.centcom.set_mission_param(self.ship, 'site', site.symbol)
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission()
def do_cprobe(self):
@ -160,15 +169,14 @@ class Commander(CommandLine):
return
markets = [ mkt[1] for mkt in m]
markets = self.analyzer.solve_tsp(markets)
hops = ','.join([m.symbol for m in markets])
self.centcom.init_mission(self.ship, 'probe')
self.centcom.set_mission_param(self.ship, 'hops', hops)
self.centcom.set_mission_param(self.ship, 'hops', markets)
self.print_mission()
def do_travel(self, dest):
dest = self.resolve('Waypoint', dest)
self.centcom.init_mission(self.ship, 'travel')
self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
self.centcom.set_mission_param(self.ship, 'dest', dest)
self.print_mission()
def do_register(self, faction):
@ -194,9 +202,10 @@ class Commander(CommandLine):
def do_waypoints(self, system_str=''):
if system_str == '':
if not self.has_ship(): return
system = self.ship.location().system()
system = self.ship.location.system
else:
system = self.store.get(System, system_str)
print(f'=== waypoints in {system}')
r = self.store.all_members(system, 'Waypoint')
for w in r:
traits = []
@ -220,7 +229,7 @@ class Commander(CommandLine):
def do_jumps(self, waypoint_str=None):
if waypoint_str is None:
if not self.has_ship(): return
waypoint = self.ship.location()
waypoint = self.ship.location
else:
waypoint = self.store.get(Waypoint, waypoint_str.upper())
r = self.api.jumps(waypoint)
@ -228,7 +237,7 @@ class Commander(CommandLine):
def do_query(self, resource):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
resource = resource.upper()
print('Found markets:')
for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@ -261,7 +270,7 @@ class Commander(CommandLine):
def do_deliver(self):
if not self.has_ship(): return
site = self.ship.location_str
site = self.ship.location
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@ -290,7 +299,7 @@ class Commander(CommandLine):
def do_go(self, arg):
if not self.has_ship(): return
system = self.ship.location().system()
system = self.ship.location.system
symbol = f'{system}-{arg}'
dest = self.resolve('Waypoint', symbol)
self.api.navigate(self.ship, dest)
@ -324,7 +333,7 @@ class Commander(CommandLine):
def do_market(self, arg=''):
if arg == '':
if not self.has_ship(): return
waypoint = self.ship.location()
waypoint = self.ship.location
else:
waypoint = self.resolve('Waypoint', arg)
r = self.api.marketplace(waypoint)
@ -354,7 +363,7 @@ class Commander(CommandLine):
def do_shipyard(self):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
data = self.api.shipyard(location)
for s in must_get(data, 'ships'):
print(s['type'], s['purchasePrice'])
@ -362,7 +371,7 @@ class Commander(CommandLine):
def do_jump(self, system_str):
if not self.has_ship(): return
if '-' not in system_str:
sector = self.ship.location_str.split('-')[0]
sector = self.ship.location.system.sector.symbol
system_str = f'{sector}-{system_str}'
system = self.resolve('System', system_str)
self.api.jump(self.ship, system)
@ -370,7 +379,7 @@ class Commander(CommandLine):
def do_purchase(self, ship_type):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
ship_type = ship_type.upper()
if not ship_type.startswith('SHIP'):
ship_type = 'SHIP_' + ship_type

View File

@ -26,9 +26,14 @@ class MissionParam:
elif self.cls == int:
return int(val)
elif self.cls == list:
return [i.strip() for i in val.split(',')]
if type(val) == str:
return [i.strip() for i in val.split(',')]
return val
elif issubclass(self.cls, Base):
data = store.get(self.cls, val)
if type(val) == str:
data = store.get(self.cls, val)
else:
data = val
if data is None:
raise ValueError('object not found')
return data.symbol
@ -179,14 +184,13 @@ class BaseMission(Mission):
traject = self.st('traject')
if traject is None or traject == []:
return 'done'
dest = self.store.get(Waypoint, traject[-1])
loc = self.ship.location()
print(dest, loc)
dest = traject[-1]
loc = self.ship.location
if dest == loc:
self.sts('traject', None)
return 'done'
hop = traject.pop(0)
if len(hop.split('-')) == 3:
if type(hop) == Waypoint:
self.api.navigate(self.ship, hop)
self.next_step = self.ship.arrival
else:
@ -200,22 +204,23 @@ class BaseMission(Mission):
def step_calculate_traject(self, dest):
if type(dest) == str:
dest = self.store.get(Waypoint, dest)
loc = self.ship.location()
loc_sys = self.store.get(System, loc.system())
loc = self.ship.location
loc_sys = loc.system
loc_jg = self.analyzer.get_jumpgate(loc_sys)
dest_sys = self.store.get(System, dest.system())
loc_jg_wp = self.store.get(Waypoint, loc_jg.symbol)
dest_sys = dest.system
dest_jg = self.analyzer.get_jumpgate(dest_sys)
if dest_sys == loc_sys:
result = [dest.symbol]
result = [dest]
self.sts('traject', result)
return
path = self.analyzer.find_path(loc_sys, dest_sys)
result = []
if loc.symbol != loc_jg.symbol:
result.append(loc_jg.symbol)
result += [s.symbol for s in path[1:]]
result.append(loc_jg_wp)
result += [s for s in path[1:]]
if dest_jg.symbol != dest.symbol:
result.append(dest.symbol)
result.append(dest)
self.sts('traject', result)
print(result)
return result

View File

@ -0,0 +1,12 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

View File

@ -8,9 +8,6 @@ class Agent(Base):
def update(self, d):
self.seta('credits', d)
def path(self):
return f'{self.symbol}.{self.ext()}'
@classmethod
def ext(self):
return 'agt'

View File

@ -1,18 +1,38 @@
from copy import deepcopy
from nullptr.util import sg
class Reference:
def __init__(self, typ, symbol, store):
self.typ = typ
self.symbol = symbol
self.store = store
@classmethod
def create(cls, obj):
o = cls(type(obj), obj.symbol, obj.store)
return o
def resolve(self):
return self.store.get(self.typ, self.symbol)
def __repr__(self):
return f'*REF*{self.symbol}.{self.typ.ext()}'
class Base:
identifier = 'symbol'
symbol: str
store: object
def __init__(self, symbol, store):
self.disable_dirty = True
self.file_offset = None
self.store = store
self.symbol = symbol
self.define()
self.disable_dirty = False
@classmethod
def ext(cls):
raise NotImplementedError('no ext')
def define(self):
pass
@ -20,7 +40,13 @@ class Base:
return hash((str(type(self)), self.symbol))
def __eq__(self, other):
return self.symbol == other.symbol and type(self) == type(other)
return type(self) == type(other) and self.symbol == other.symbol
def get_system(self):
parts = self.symbol.split('-')
system_str = f'{parts[0]}-{parts[1]}'
system = self.store.get('System', system_str, create=True)
return system
def seta(self, attr, d, name=None, interp=None):
if name is None:
@ -31,17 +57,30 @@ class Base:
val = interp(val)
setattr(self, attr, val)
def setlst(self, attr, d, name, member):
def setlst(self, attr, d, name, member, interp=None):
val = sg(d, name)
if val is not None:
lst = [sg(x, member) for x in val]
lst = []
for x in val:
val = sg(x, member)
if interp is not None:
val = interp(val)
lst.append(val)
setattr(self, attr, lst)
def __setattr__(self, name, value):
if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
self.store.dirty(self)
if issubclass(type(value), Base):
value = Reference.create(value)
super().__setattr__(name, value)
def __getattribute__(self, nm):
val = super().__getattribute__(nm)
if type(val) == Reference:
val = val.resolve()
return val
def update(self, d):
pass
@ -53,27 +92,15 @@ class Base:
self.__dict__.update(d)
self.disable_dirty = False
def dict(self):
r = {}
for k,v in self.__dict__.items():
if k in ['store']:
continue
r[k] = deepcopy(v)
return r
def path(self):
raise NotImplementedError('path')
@classmethod
def ext(self):
raise NotImplementedError('extension')
def type(self):
return self.__class__.__name__
def __str__(self):
return self.f()
def __repr__(self):
return self.f()
def f(self, detail=1):
r = self.symbol
if detail > 1:

View File

@ -18,9 +18,6 @@ class Contract(Base):
def ext(cls):
return 'cnt'
def path(self):
return f'contracts/{self.symbol}.{self.ext()}'
def is_expired(self):
return time() > self.expires

View File

@ -1,14 +1,17 @@
from .system_member import SystemMember
from .base import Base
from .system import System
from dataclasses import field
class Jumpgate(SystemMember):
class Jumpgate(Base):
def define(self):
self.range: int = 0
self.faction: str = ''
self.systems: list = []
self.system = self.get_system()
def update(self, d):
self.setlst('systems', d, 'connectedSystems', 'symbol')
getter = self.store.getter(System, create=True)
self.setlst('systems', d, 'connectedSystems', 'symbol', interp=getter)
self.seta('faction', d, 'factionSymbol')
self.seta('range', d, 'jumpRange')
@ -16,13 +19,9 @@ class Jumpgate(SystemMember):
def ext(self):
return 'jmp'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def f(self, detail=1):
r = self.symbol
if detail > 1:
r += '\n'
r += '\n'.join(self.systems)
r += '\n'.join([s.symbol for s in self.systems])
return r

View File

@ -1,16 +1,23 @@
from .system_member import SystemMember
from .base import Base
from time import time
from nullptr.util import *
from dataclasses import field
from nullptr.models import Waypoint
class Marketplace(SystemMember):
class Marketplace(Base):
def define(self):
self.imports:list = []
self.exports:list = []
self.exchange:list = []
self.prices:dict = {}
self.last_prices:int = 0
self.set_waypoint()
self.system = self.get_system()
def set_waypoint(self):
waypoint = self.store.get(Waypoint, self.symbol, create=True)
self.waypoint = waypoint
def update(self, d):
self.setlst('imports', d, 'imports', 'symbol')
@ -44,10 +51,6 @@ class Marketplace(SystemMember):
if r in self.exchange:
return 'X'
return '?'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def f(self, detail=1):
r = self.symbol

View File

@ -1,7 +1,7 @@
from .base import Base
from time import time
from nullptr.util import *
from dataclasses import dataclass, field
from nullptr.models import Waypoint
class Ship(Base):
def define(self):
@ -10,7 +10,7 @@ class Ship(Base):
self.status:str = ''
self.cargo_capacity:int = 0
self.cargo_units:int = 0
self.location_str = ''
self.location = None
self.cooldown:int = 0
self.arrival:int = 0
self.fuel_current:int = 0
@ -22,16 +22,10 @@ class Ship(Base):
def ext(self):
return 'shp'
def location(self):
return self.store.get('Waypoint', self.location_str)
def path(self):
agent = self.symbol.split('-')[0]
return f'{agent}/{self.symbol}.{self.ext()}'
def update(self, d):
self.seta('status', d, 'nav.status')
self.seta('location_str', d, 'nav.waypointSymbol')
getter = self.store.getter(Waypoint, create=True)
self.seta('location', d, 'nav.waypointSymbol', interp=getter)
self.seta('cargo_capacity', d, 'cargo.capacity')
self.seta('cargo_units', d, 'cargo.units')
self.seta('fuel_capacity', d, 'fuel.capacity')
@ -95,7 +89,7 @@ class Ship(Base):
if detail > 1:
r += ' ' + self.status
r += f' [{self.fuel_current}/{self.fuel_capacity}]'
r += ' ' + str(self.location())
r += ' ' + str(self.location)
if self.is_travelling():
r += f' [A: {arrival}]'
if self.is_cooldown():

View File

@ -1,10 +1,10 @@
from time import time
from nullptr.util import *
from .system_member import SystemMember
from .base import Base
size_names = ['SMALL','MODERATE','LARGE']
class Survey(SystemMember):
class Survey(Base):
identifier = 'signature'
def define(self):
self.type: str = ''
@ -17,12 +17,7 @@ class Survey(SystemMember):
@classmethod
def ext(cls):
return 'svy'
def path(self):
sector, system, waypoint, signature = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def is_expired(self):
return time() > self.expires or self.exhausted

View File

@ -17,10 +17,6 @@ class System(Base):
def ext(self):
return 'stm'
def path(self):
sector, symbol = self.symbol.split('-')
return f'atlas/{sector}/{symbol[0:1]}/{self.symbol}.{self.ext()}'
def distance(self, other):
return int(sqrt((self.x - other.x) ** 2 + (self.y - other.y) ** 2))

View File

@ -1,10 +0,0 @@
from .base import Base
class SystemMember(Base):
@classmethod
def ext(cls):
return 'obj'
def system(self):
p = self.symbol.split('-')
return f'{p[0]}-{p[1]}'

View File

@ -1,26 +1,24 @@
from .system_member import SystemMember
from .base import Base, Reference
from nullptr.models.system import System
from nullptr.util import *
from dataclasses import field
class Waypoint(SystemMember):
class Waypoint(Base):
def define(self):
self.x:int = 0
self.y:int = 0
self.type:str = 'unknown'
self.traits:list = []
self.faction:str = ''
self.system = self.get_system()
def update(self, d):
self.seta('x', d)
self.seta('y', d)
self.seta('type', d)
self.seta('faction', d, 'faction.symbol')
self.setlst('traits', d, 'traits', 'symbol')
@classmethod
def ext(self):
return 'way'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'

View File

@ -1,30 +1,74 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.system_member import SystemMember
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
from nullptr.models import *
from os.path import isfile, dirname, isdir
import os
from os.path import basename
import json
from .util import *
from time import time
import pickle
from struct import unpack, pack
from functools import partial
from io import BytesIO
class StorePickler(pickle.Pickler):
def persistent_id(self, obj):
return "STORE" if type(obj) == Store else None
class StoreUnpickler(pickle.Unpickler):
def __init__(self, stream, store):
self.store = store
super().__init__(stream)
def persistent_load(self, pers_id):
if pers_id == "STORE":
return self.store
raise pickle.UnpicklingError("I don know the persid!")
class ChunkHeader:
def __init__(self):
self.offset = 0
self.in_use = True
self.size = 0
self.used = 0
@classmethod
def parse(cls, fil):
offset = fil.tell()
d = fil.read(16)
if len(d) < 16:
return None
o = cls()
o.offset = offset
d, o.used = unpack('<QQ', d)
o.size = d & 0x7fffffffffffffff
o.in_use = d & 0x8000000000000000 != 0
# print(o)
return o
def write(self, f):
d = self.size
if self.in_use:
d |= 1 << 63
d = pack('<QQ', d, self.used)
f.write(d)
def __repr__(self):
return f'chunk {self.in_use} {self.size} {self.used}'
class Store:
def __init__(self, data_dir):
def __init__(self, data_file):
self.init_models()
self.data_dir = data_dir
self.fil = open_file(data_file)
self.data = {m: {} for m in self.models}
self.system_members = {}
self.dirty_objects = set()
self.cleanup_interval = 600
self.last_cleanup = 0
self.slack = 0.1
self.slack_min = 64
self.slack_max = 1024
self.load()
def init_models(self):
self.models = all_subclasses(Base)
@ -33,54 +77,99 @@ class Store:
def dirty(self, obj):
self.dirty_objects.add(obj)
def path(self, obj):
return os.path.join(self.data_dir, obj.path())
def load_file(self, path):
if not isfile(path):
return None
fn = basename(path)
ext = fn.split('.')[-1]
symbol = fn.split('.')[0]
if ext not in self.extensions:
return None
with open(path) as f:
data = json.load(f)
typ = self.extensions[ext]
obj = self.create(typ, symbol)
obj.load(data)
obj.store = self
return obj
def dump_object(self, obj):
buf = BytesIO()
p = StorePickler(buf)
p.dump(obj)
return buf.getvalue()
def load_object(self, data, offset):
buf = BytesIO(data)
p = StoreUnpickler(buf, self)
obj = p.load()
obj.file_offset = offset
obj.disable_dirty = False
self.hold(obj)
def load(self):
cnt = 0
start_time = time()
for fil in list_files(self.data_dir, True):
self.load_file(fil)
self.fil.seek(0)
offset = 0
while (hdr := ChunkHeader.parse(self.fil)):
# print(hdr)
if not hdr.in_use:
self.fil.seek(hdr.size, 1)
continue
data = self.fil.read(hdr.used)
self.load_object(data, offset)
self.fil.seek(hdr.size - hdr.used, 1)
offset = self.fil.tell()
cnt += 1
dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds')
def allocate_chunk(self, sz):
used = sz
slack = sz * self.slack
slack = min(slack, self.slack_max)
slack = max(slack, self.slack_min)
sz += int(slack)
self.fil.seek(0, 2)
offset = self.fil.tell()
h = ChunkHeader()
h.size = sz
h.used = used
h.offset = self.fil.tell()
h.write(self.fil)
return offset, h
def store(self, obj):
path = self.path(obj)
path_dir = dirname(path)
data = obj.dict()
if not isdir(path_dir):
os.makedirs(path_dir, exist_ok=True)
with open(path, 'w') as f:
json.dump(data, f, indent=2)
def create(self, typ, symbol):
obj = typ(symbol, self)
data = self.dump_object(obj)
osize = len(data)
# is there an existing chunk for this obj?
if obj.file_offset is not None:
# read chunk hdr
self.fil.seek(obj.file_offset)
hdr = ChunkHeader.parse(self.fil)
csize = hdr.size
# if the chunk is too small
if csize < osize:
# free the chunk
hdr.in_use = False
# force a new chunk
obj.file_offset = None
else:
# if it is big enough, update the used field
hdr.used = osize
self.fil.seek(hdr.offset)
hdr.write(self.fil)
if obj.file_offset is None:
obj.file_offset, hdr = self.allocate_chunk(osize)
# print(type(obj).__name__, hdr)
self.fil.write(data)
slack = b'\x00' * (hdr.size - hdr.used)
self.fil.write(slack)
def hold(self, obj):
typ = type(obj)
symbol = obj.symbol
obj.store = self
self.data[typ][symbol] = obj
if issubclass(typ, SystemMember):
system_str = obj.system()
if hasattr(obj, 'system') and obj.system != None:
system_str = obj.system.symbol
if system_str not in self.system_members:
self.system_members[system_str] = set()
self.system_members[system_str].add(obj)
def create(self, typ, symbol):
obj = typ(symbol, self)
self.hold(obj)
self.dirty(obj)
return obj
def get(self, typ, symbol, create=False):
@ -96,6 +185,11 @@ class Store:
return None
return self.data[typ][symbol]
def getter(self, typ, create=False):
if type(typ) == str and typ in self.model_names:
typ = self.model_names[typ]
return partial(self.get, typ, create=create)
def update(self, typ, data, symbol=None):
if type(typ) == str and typ in self.model_names:
typ = self.model_names[typ]
@ -121,10 +215,10 @@ class Store:
if type(system) == System:
system = system.symbol
if system not in self.system_members:
return
print('typ', typ)
for m in self.system_members[system]:
if typ is None or type(m) == typ:
yield m
@ -139,9 +233,9 @@ class Store:
if o.is_expired():
expired.append(o)
for o in expired:
path = o.path()
if isfile(path):
os.remove(path)
# TODO
del self.data[type(o)][o.symbol]
dur = time() - start_time
# print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@ -153,6 +247,7 @@ class Store:
for obj in self.dirty_objects:
it += 1
self.store(obj)
self.fil.flush()
self.dirty_objects = set()
dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}')

View File

@ -1,21 +1,16 @@
from datetime import datetime
from math import ceil
import os
from os.path import isfile
from os.path import isfile, dirname
def list_files(path, recursive=False):
if recursive:
for p, dirnames, fils in os.walk(path):
for f in fils:
fil = os.path.join(p, f)
yield fil
def open_file(fn):
d = dirname(fn)
os.makedirs(d, exist_ok=True)
if isfile(fn):
return open(fn, 'rb+')
else:
for f in os.listdir(path):
fil = os.path.join(path, f)
if not isfile(fil):
continue
yield fil
return open(fn, 'ab+')
def must_get(d, k):
if type(k) == str:
k = k.split('.')
@ -58,7 +53,7 @@ def pretty(d, ident=0, detail=2):
return d.f(detail)
r = ''
idt = ' ' * ident
if type(d) == list:
if type(d) in [list, set]:
r += 'lst'
for i in d:
r += '\n' + idt + pretty(i, ident + 1, detail)

View File

@ -3,6 +3,10 @@ This project uses a custom database format just because we can.
The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
The store disk format is optimized for two things:
* Loading the entire contents in memory
* Writing changed objects back to disk
## Objects
First lets discuss what we are storing.
@ -30,12 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict
* store.load(fil) loads all objects
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
* store.all(type) generator for all objects of a goven type
* store.delete(typ, symbol)
* store.cleanup() removes all expired objects
* store.flush() writes all dirty objects to disk
*
* store.defrag() consolidates the store file to minimize storage and loading time
type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
# file format
the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header.
Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.