richard/0ptr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: richard:integrated-atlas-builder
Branches Tags
richard:master richard:newstore
richard:integrated-atlas-builder richard:single-file-store richard:hauling richard:mining-again richard:market-info
..
compare: richard:5d0595967de8c3dfb847ed51f917f5af162d72f6
Branches Tags
richard:master richard:newstore
richard:integrated-atlas-builder richard:single-file-store richard:hauling richard:mining-again richard:market-info

1 Commits
integrated ... 5d0595967d

Author SHA1 Message Date
Richard Bronkhorst
5d0595967d Update central_command.py, commander.py and one other file 2023-07-03 19:12:26 +02:00
23 changed files with 267 additions and 502 deletions
Expand all files Collapse all files

2
Dockerfile
View File

@@ -9,4 +9,4 @@ ADD --chown=user . /app
RUN chmod +x /app/main.py
VOLUME /data
ENTRYPOINT [ "python3", "/app/main.py"]
CMD ["-s", "/data/store.npt"]
CMD ["-s", "/data/"]

7
main.py Executable file → Normal file
View File

@@ -1,9 +1,8 @@
#!/usr/bin/env python3
import argparse
from nullptr.commander import Commander
from nullptr.models.base import Base
def main(args):
c = Commander(args.store_file)
c = Commander(args.store_dir)
c.run()
# X1-AG74-41076A
@@ -11,6 +10,6 @@ def main(args):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--store-file', default='data/store.npt')
parser.add_argument('-s', '--store-dir', default='data')
args = parser.parse_args()
main(args)

10
nullptr/analyzer.py
View File

@@ -44,16 +44,16 @@ class Analyzer:
location = self.store.get(Waypoint, location)
mkts = self.find_markets(resource, sellbuy)
candidates = []
origin = location.system
origin = self.store.get(System, location.system())
for typ, m in mkts:
system = m.waypoint.system
system = self.store.get(System, m.system())
d = origin.distance(system)
candidates.append((typ, m, d))
possibles = sorted(candidates, key=lambda m: m[2])
possibles = possibles[:10]
results = []
for typ,m,d in possibles:
system = m.waypoint.system
system = self.store.get(System, m.system())
p = self.find_path(origin, system)
if p is None: continue
results.append((typ,m,d,len(p)))
@@ -83,7 +83,9 @@ class Analyzer:
for s in jg.systems:
if s in seen: continue
seen.add(s)
dest.add(SearchNode(s, o))
system = self.store.get(System, s)
if system is None: continue
dest.add(SearchNode(system, o))
if len(dest) == 0:
return None
return self.find_path(dest, to, depth-1, seen)

15
nullptr/api.py
View File

@@ -19,7 +19,7 @@ class Api:
self.agent = agent
self.store = store
self.requests_sent = 0
self.last_meta = None
self.meta = None
self.last_result = None
self.root = 'https://api.spacetraders.io/v2/'
@@ -69,7 +69,6 @@ class Api:
}
result = self.request('post', 'register', data, need_token=False)
token = mg(result, 'token')
self.agent.update(mg(result, 'agent'))
self.agent.token = token
def info(self):
@@ -80,10 +79,7 @@ class Api:
def list_systems(self, page=1):
data = self.request('get', 'systems', params={'page': page})
#pprint(self.last_meta)
systems = self.store.update_list(System, data)
for s in data:
self.store.update_list(Waypoint, mg(s, 'waypoints'))
return systems
return self.store.update_list(System, data)
def list_waypoints(self, system):
data = self.request('get', f'systems/{system}/waypoints/')
@@ -91,12 +87,13 @@ class Api:
return self.store.update_list(Waypoint, data)
def marketplace(self, waypoint):
system = waypoint.system
system = waypoint.system()
symbol = str(waypoint)
data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
return self.store.update(Marketplace, data)
def jumps(self, waypoint):
data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
symbol = str(waypoint)
return self.store.update(Jumpgate, data, symbol)
@@ -235,7 +232,7 @@ class Api:
return ship
def shipyard(self, wp):
return self.request('get', f'systems/{wp.system}/waypoints/{wp}/shipyard')
return self.request('get', f'systems/{wp.system()}/waypoints/{wp}/shipyard')
def extract(self, ship, survey=None):
data = {}

107
nullptr/atlas_builder.py
View File

@@ -1,71 +1,66 @@
from time import sleep, time
from time import sleep
from nullptr.util import *
from threading import Thread
from nullptr.models.atlas import Atlas
from functools import partial
from nullptr.models import System
class AtlasBuilder:
def __init__(self, store, api):
self.store = store
self.api = api
self.work = []
self.max_work = 100
self.unch_interval = 86400
self.atlas = self.store.get(Atlas, 'ATLAS', create=True)
self.stop_auto = False
def find_work(self):
first_page = self.atlas.total_pages == 0
pages_left = self.atlas.total_pages < self.atlas.seen_pages
if first_page or pages_left:
self.sched(self.get_systems)
return
for s in self.store.all(System):
if len(self.work) > self.max_work:
break
if not s.uncharted: continue
if s.last_crawl > time() - self.unch_interval:
continue
self.sched(self.get_waypoints, s)
def wait_for_stop(self):
try:
input()
except EOFError:
pass
self.stop_auto = True
print('stopping...')
def run(self, page=1):
print('universe mode. hit enter to stop')
t = Thread(target=self.wait_for_stop)
t.daemon = True
t.start()
self.all_systems(int(page))
print('manual mode')
def do_work(self):
if len(self.work) == 0:
self.find_work()
if len(self.work) == 0:
return
work = self.work.pop()
work()
def get_systems(self):
page = 1
if self.atlas.seen_pages > 0:
page = self.atlas.seen_pages + 1
if page > self.atlas.total_pages:
return
data = self.api.list_systems(page)
self.atlas.total_pages = total_pages(self.api.last_meta)
self.atlas.seen_pages = page
def get_waypoints(self, system):
wps = self.api.list_waypoints(system)
system.last_crawl = time()
system.uncharted = len([1 for w in wps if w.uncharted]) > 0
self.schedule_specials(wps)
def sched(self, fun, *args):
self.work.append(partial(fun, *args))
def schedule_specials(self, waypoints):
def all_specials(self, waypoints):
for w in waypoints:
if self.stop_auto:
break
if 'UNCHARTED' in w.traits:
continue
if 'MARKETPLACE' in w.traits:
# print(f'marketplace at {w}')
self.sched(self.api.marketplace, w)
self.api.marketplace(w)
print(f'marketplace at {w}')
sleep(0.5)
if w.type == 'JUMP_GATE':
# print(f'jumpgate at {w}')
self.sched(self.api.jumps, w)
if 'SHIPYARD' in w.traits:
# todo
pass
self.api.jumps(w)
print(f'jumpgate at {w}')
def all_waypoints(self, systems):
for s in systems:
if self.stop_auto:
break
r = self.api.list_waypoints(s)
self.all_specials(r)
sleep(0.5)
def all_systems(self, start_page):
self.stop_auto = False
data = self.api.list_systems(start_page)
pages = total_pages(self.api.last_meta)
print(f'{pages} pages of systems')
print(f'page {1}: {len(data)} results')
self.all_waypoints(data)
self.store.flush()
for p in range(start_page+1, pages+1):
if self.stop_auto:
break
data = self.api.list_systems(p)
print(f'page {p}: {len(data)} systems')
self.all_waypoints(data)
sleep(0.5)
self.store.flush()

5
nullptr/central_command.py
View File

@@ -4,7 +4,6 @@ from nullptr.missions import create_mission, get_mission_class
from random import choice
from time import sleep
from threading import Thread
from nullptr.atlas_builder import AtlasBuilder
class CentralCommandError(Exception):
pass
@@ -15,7 +14,6 @@ class CentralCommand:
self.stopping = False
self.store = store
self.api = api
self.atlas_builder = AtlasBuilder(store, api)
self.update_missions()
def get_ready_missions(self):
@@ -58,9 +56,6 @@ class CentralCommand:
request_counter = self.api.requests_sent
while request_counter == self.api.requests_sent and did_step:
did_step = self.tick()
if request_counter == self.api.requests_sent:
self.atlas_builder.do_work()
self.store.flush()
sleep(0.5)
self.stopping = False

78
nullptr/commander.py
View File

@@ -2,21 +2,28 @@ from nullptr.command_line import CommandLine
from nullptr.store import Store
from nullptr.analyzer import Analyzer
import argparse
from nullptr.models import *
from nullptr.models.agent import Agent
from nullptr.models.system import System
from nullptr.models.waypoint import Waypoint
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.api import Api
from .util import *
from time import sleep, time
from threading import Thread
from nullptr.atlas_builder import AtlasBuilder
from nullptr.central_command import CentralCommand
class CommandError(Exception):
pass
class Commander(CommandLine):
def __init__(self, store_file='data/store.npt'):
self.store = Store(store_file)
def __init__(self, store_dir='data'):
self.store_dir = store_dir
self.store = Store(store_dir)
self.store.load()
self.agent = self.select_agent()
self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api)
self.centcom = CentralCommand(self.store, self.api)
self.analyzer = Analyzer(self.store)
self.ship = None
@@ -49,26 +56,8 @@ class Commander(CommandLine):
agents = self.store.all(Agent)
agent = next(agents, None)
if agent is None:
agent = self.agent_setup()
return agent
def agent_setup(self):
symbol = input('agent name: ')
agent = self.store.get(Agent, symbol, create=True)
self.agent = agent
api = Api(self.store, agent)
self.api = api
faction = input('faction: ')
api.register(faction.upper().strip())
print('=== agent:')
print(agent)
print('=== ships')
self.do_ships('r')
print('=== contracts')
self.do_contracts('r')
ship = self.store.get(Ship, symbol.upper() + '-2')
api.list_waypoints(ship.location.system)
self.store.flush()
return agent
def resolve(self, typ, arg):
@@ -118,7 +107,7 @@ class Commander(CommandLine):
def do_cmine(self):
if not self.has_ship(): return
site = self.ship.location
site = self.ship.location_str
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@@ -129,12 +118,12 @@ class Commander(CommandLine):
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.print_mission()
def do_chaul(self):
if not self.has_ship(): return
if len(self.ship.cargo) > 0:
if len(ship.cargo) > 0:
raise CommandError('please dump cargo first')
contract = self.active_contract()
delivery = contract.unfinished_delivery()
@@ -151,10 +140,10 @@ class Commander(CommandLine):
_, m, _, _ = m[0]
site = self.store.get(Waypoint, m.symbol)
self.centcom.init_mission(self.ship, 'haul')
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'site', site.symbol)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.print_mission()
def do_cprobe(self):
@@ -171,20 +160,24 @@ class Commander(CommandLine):
return
markets = [ mkt[1] for mkt in m]
markets = self.analyzer.solve_tsp(markets)
hops = ','.join([m.symbol for m in markets])
self.centcom.init_mission(self.ship, 'probe')
self.centcom.set_mission_param(self.ship, 'hops', markets)
self.centcom.set_mission_param(self.ship, 'hops', hops)
self.print_mission()
def do_travel(self, dest):
dest = self.resolve('Waypoint', dest)
self.centcom.init_mission(self.ship, 'travel')
self.centcom.set_mission_param(self.ship, 'dest', dest)
self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
self.print_mission()
def do_register(self, faction):
self.api.register(faction.upper())
pprint(self.api.agent)
def do_universe(self, page=1):
self.atlas_builder.run(page)
def do_systems(self, page=1):
r = self.api.list_systems(int(page))
pprint(self.api.last_meta)
@@ -198,21 +191,12 @@ class Commander(CommandLine):
print(f'{num:5d} {nam}')
print(f'{total:5d} total')
def do_defrag(self):
self.store.defrag()
def do_system(self, system_str):
system = self.store.get(System, system_str)
r = self.api.list_waypoints(system)
pprint(r)
def do_waypoints(self, system_str=''):
if system_str == '':
if not self.has_ship(): return
system = self.ship.location.system
system = self.ship.location().system()
else:
system = self.store.get(System, system_str)
print(f'=== waypoints in {system}')
r = self.store.all_members(system, 'Waypoint')
for w in r:
traits = []
@@ -236,7 +220,7 @@ class Commander(CommandLine):
def do_jumps(self, waypoint_str=None):
if waypoint_str is None:
if not self.has_ship(): return
waypoint = self.ship.location
waypoint = self.ship.location()
else:
waypoint = self.store.get(Waypoint, waypoint_str.upper())
r = self.api.jumps(waypoint)
@@ -244,7 +228,7 @@ class Commander(CommandLine):
def do_query(self, resource):
if not self.has_ship(): return
location = self.ship.location
location = self.ship.location()
resource = resource.upper()
print('Found markets:')
for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@@ -277,7 +261,7 @@ class Commander(CommandLine):
def do_deliver(self):
if not self.has_ship(): return
site = self.ship.location
site = self.ship.location_str
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@@ -306,7 +290,7 @@ class Commander(CommandLine):
def do_go(self, arg):
if not self.has_ship(): return
system = self.ship.location.system
system = self.ship.location().system()
symbol = f'{system}-{arg}'
dest = self.resolve('Waypoint', symbol)
self.api.navigate(self.ship, dest)
@@ -340,7 +324,7 @@ class Commander(CommandLine):
def do_market(self, arg=''):
if arg == '':
if not self.has_ship(): return
waypoint = self.ship.location
waypoint = self.ship.location()
else:
waypoint = self.resolve('Waypoint', arg)
r = self.api.marketplace(waypoint)
@@ -370,7 +354,7 @@ class Commander(CommandLine):
def do_shipyard(self):
if not self.has_ship(): return
location = self.ship.location
location = self.ship.location()
data = self.api.shipyard(location)
for s in must_get(data, 'ships'):
print(s['type'], s['purchasePrice'])
@@ -378,7 +362,7 @@ class Commander(CommandLine):
def do_jump(self, system_str):
if not self.has_ship(): return
if '-' not in system_str:
sector = self.ship.location.system.sector.symbol
sector = self.ship.location_str.split('-')[0]
system_str = f'{sector}-{system_str}'
system = self.resolve('System', system_str)
self.api.jump(self.ship, system)
@@ -386,7 +370,7 @@ class Commander(CommandLine):
def do_purchase(self, ship_type):
if not self.has_ship(): return
location = self.ship.location
location = self.ship.location()
ship_type = ship_type.upper()
if not ship_type.startswith('SHIP'):
ship_type = 'SHIP_' + ship_type

29
nullptr/missions/base.py
View File

@@ -26,14 +26,9 @@ class MissionParam:
elif self.cls == int:
return int(val)
elif self.cls == list:
if type(val) == str:
return [i.strip() for i in val.split(',')]
return val
elif issubclass(self.cls, Base):
if type(val) == str:
data = store.get(self.cls, val)
else:
data = val
if data is None:
raise ValueError('object not found')
return data.symbol
@@ -163,7 +158,7 @@ class BaseMission(Mission):
def step_sell(self, except_resource=True):
target = self.st('resource')
market = self.store.get('Marketplace', self.ship.location.symbol)
market = self.store.get('Marketplace', self.ship.location_str)
sellables = market.sellable_items(self.ship.cargo.keys())
if target in sellables and except_resource:
sellables.remove(target)
@@ -184,13 +179,14 @@ class BaseMission(Mission):
traject = self.st('traject')
if traject is None or traject == []:
return 'done'
dest = traject[-1]
loc = self.ship.location
dest = self.store.get(Waypoint, traject[-1])
loc = self.ship.location()
print(dest, loc)
if dest == loc:
self.sts('traject', None)
return 'done'
hop = traject.pop(0)
if type(hop) == Waypoint:
if len(hop.split('-')) == 3:
self.api.navigate(self.ship, hop)
self.next_step = self.ship.arrival
else:
@@ -204,23 +200,22 @@ class BaseMission(Mission):
def step_calculate_traject(self, dest):
if type(dest) == str:
dest = self.store.get(Waypoint, dest)
loc = self.ship.location
loc_sys = loc.system
loc = self.ship.location()
loc_sys = self.store.get(System, loc.system())
loc_jg = self.analyzer.get_jumpgate(loc_sys)
loc_jg_wp = self.store.get(Waypoint, loc_jg.symbol)
dest_sys = dest.system
dest_sys = self.store.get(System, dest.system())
dest_jg = self.analyzer.get_jumpgate(dest_sys)
if dest_sys == loc_sys:
result = [dest]
result = [dest.symbol]
self.sts('traject', result)
return
path = self.analyzer.find_path(loc_sys, dest_sys)
result = []
if loc.symbol != loc_jg.symbol:
result.append(loc_jg_wp)
result += [s for s in path[1:]]
result.append(loc_jg.symbol)
result += [s.symbol for s in path[1:]]
if dest_jg.symbol != dest.symbol:
result.append(dest)
result.append(dest.symbol)
self.sts('traject', result)
print(result)
return result

13
nullptr/models/__init__.py
View File

@@ -1,13 +0,0 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
from nullptr.models.atlas import Atlas
__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base', 'Atlas' ]

3
nullptr/models/agent.py
View File

@@ -8,6 +8,9 @@ class Agent(Base):
def update(self, d):
self.seta('credits', d)
def path(self):
return f'{self.symbol}.{self.ext()}'
@classmethod
def ext(self):
return 'agt'

11
nullptr/models/atlas.py
View File

@@ -1,11 +0,0 @@
from .base import Base
class Atlas(Base):
@classmethod
def ext(self):
return 'atl'
def define(self):
self.total_pages = 0
self.seen_pages = 0

69
nullptr/models/base.py
View File

@@ -1,38 +1,18 @@
from copy import deepcopy
from nullptr.util import sg
class Reference:
def __init__(self, typ, symbol, store):
self.typ = typ
self.symbol = symbol
self.store = store
@classmethod
def create(cls, obj):
o = cls(type(obj), obj.symbol, obj.store)
return o
def resolve(self):
return self.store.get(self.typ, self.symbol)
def __repr__(self):
return f'*REF*{self.symbol}.{self.typ.ext()}'
class Base:
identifier = 'symbol'
symbol: str
store: object
def __init__(self, symbol, store):
self.disable_dirty = True
self.file_offset = None
self.store = store
self.symbol = symbol
self.define()
self.disable_dirty = False
@classmethod
def ext(cls):
raise NotImplementedError('no ext')
def define(self):
pass
@@ -40,13 +20,7 @@ class Base:
return hash((str(type(self)), self.symbol))
def __eq__(self, other):
return type(self) == type(other) and self.symbol == other.symbol
def get_system(self):
parts = self.symbol.split('-')
system_str = f'{parts[0]}-{parts[1]}'
system = self.store.get('System', system_str, create=True)
return system
return self.symbol == other.symbol and type(self) == type(other)
def seta(self, attr, d, name=None, interp=None):
if name is None:
@@ -57,30 +31,17 @@ class Base:
val = interp(val)
setattr(self, attr, val)
def setlst(self, attr, d, name, member, interp=None):
def setlst(self, attr, d, name, member):
val = sg(d, name)
if val is not None:
lst = []
for x in val:
val = sg(x, member)
if interp is not None:
val = interp(val)
lst.append(val)
lst = [sg(x, member) for x in val]
setattr(self, attr, lst)
def __setattr__(self, name, value):
if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
self.store.dirty(self)
if issubclass(type(value), Base):
value = Reference.create(value)
super().__setattr__(name, value)
def __getattribute__(self, nm):
val = super().__getattribute__(nm)
if type(val) == Reference:
val = val.resolve()
return val
def update(self, d):
pass
@@ -92,15 +53,27 @@ class Base:
self.__dict__.update(d)
self.disable_dirty = False
def dict(self):
r = {}
for k,v in self.__dict__.items():
if k in ['store']:
continue
r[k] = deepcopy(v)
return r
def path(self):
raise NotImplementedError('path')
@classmethod
def ext(self):
raise NotImplementedError('extension')
def type(self):
return self.__class__.__name__
def __str__(self):
return self.f()
def __repr__(self):
return self.f()
def f(self, detail=1):
r = self.symbol
if detail > 1:

3
nullptr/models/contract.py
View File

@@ -18,6 +18,9 @@ class Contract(Base):
def ext(cls):
return 'cnt'
def path(self):
return f'contracts/{self.symbol}.{self.ext()}'
def is_expired(self):
return time() > self.expires

15
nullptr/models/jumpgate.py
View File

@@ -1,17 +1,14 @@
from .base import Base
from .system import System
from .system_member import SystemMember
from dataclasses import field
class Jumpgate(Base):
class Jumpgate(SystemMember):
def define(self):
self.range: int = 0
self.faction: str = ''
self.systems: list = []
self.system = self.get_system()
def update(self, d):
getter = self.store.getter(System, create=True)
self.setlst('systems', d, 'connectedSystems', 'symbol', interp=getter)
self.setlst('systems', d, 'connectedSystems', 'symbol')
self.seta('faction', d, 'factionSymbol')
self.seta('range', d, 'jumpRange')
@@ -19,9 +16,13 @@ class Jumpgate(Base):
def ext(self):
return 'jmp'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def f(self, detail=1):
r = self.symbol
if detail > 1:
r += '\n'
r += '\n'.join([s.symbol for s in self.systems])
r += '\n'.join(self.systems)
return r

15
nullptr/models/marketplace.py
View File

@@ -1,23 +1,16 @@
from .base import Base
from .system_member import SystemMember
from time import time
from nullptr.util import *
from dataclasses import field
from nullptr.models import Waypoint
class Marketplace(Base):
class Marketplace(SystemMember):
def define(self):
self.imports:list = []
self.exports:list = []
self.exchange:list = []
self.prices:dict = {}
self.last_prices:int = 0
self.set_waypoint()
self.system = self.get_system()
def set_waypoint(self):
waypoint = self.store.get(Waypoint, self.symbol, create=True)
self.waypoint = waypoint
def update(self, d):
self.setlst('imports', d, 'imports', 'symbol')
@@ -52,6 +45,10 @@ class Marketplace(Base):
return 'X'
return '?'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def f(self, detail=1):
r = self.symbol
if detail > 1:

16
nullptr/models/ship.py
View File

@@ -1,7 +1,7 @@
from .base import Base
from time import time
from nullptr.util import *
from nullptr.models import Waypoint
from dataclasses import dataclass, field
class Ship(Base):
def define(self):
@@ -10,7 +10,7 @@ class Ship(Base):
self.status:str = ''
self.cargo_capacity:int = 0
self.cargo_units:int = 0
self.location = None
self.location_str = ''
self.cooldown:int = 0
self.arrival:int = 0
self.fuel_current:int = 0
@@ -22,10 +22,16 @@ class Ship(Base):
def ext(self):
return 'shp'
def location(self):
return self.store.get('Waypoint', self.location_str)
def path(self):
agent = self.symbol.split('-')[0]
return f'{agent}/{self.symbol}.{self.ext()}'
def update(self, d):
self.seta('status', d, 'nav.status')
getter = self.store.getter(Waypoint, create=True)
self.seta('location', d, 'nav.waypointSymbol', interp=getter)
self.seta('location_str', d, 'nav.waypointSymbol')
self.seta('cargo_capacity', d, 'cargo.capacity')
self.seta('cargo_units', d, 'cargo.units')
self.seta('fuel_capacity', d, 'fuel.capacity')
@@ -89,7 +95,7 @@ class Ship(Base):
if detail > 1:
r += ' ' + self.status
r += f' [{self.fuel_current}/{self.fuel_capacity}]'
r += ' ' + str(self.location)
r += ' ' + str(self.location())
if self.is_travelling():
r += f' [A: {arrival}]'
if self.is_cooldown():

9
nullptr/models/survey.py
View File

@@ -1,10 +1,10 @@
from time import time
from nullptr.util import *
from .base import Base
from .system_member import SystemMember
size_names = ['SMALL','MODERATE','LARGE']
class Survey(Base):
class Survey(SystemMember):
identifier = 'signature'
def define(self):
self.type: str = ''
@@ -18,6 +18,11 @@ class Survey(Base):
def ext(cls):
return 'svy'
def path(self):
sector, system, waypoint, signature = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'
def is_expired(self):
return time() > self.expires or self.exhausted

6
nullptr/models/system.py
View File

@@ -7,8 +7,6 @@ class System(Base):
self.x:int = 0
self.y:int = 0
self.type:str = 'unknown'
self.uncharted = True
self.last_crawl = 0
def update(self, d):
self.seta('x', d)
@@ -19,6 +17,10 @@ class System(Base):
def ext(self):
return 'stm'
def path(self):
sector, symbol = self.symbol.split('-')
return f'atlas/{sector}/{symbol[0:1]}/{self.symbol}.{self.ext()}'
def distance(self, other):
return int(sqrt((self.x - other.x) ** 2 + (self.y - other.y) ** 2))

10
nullptr/models/system_member.py Normal file
View File

@@ -0,0 +1,10 @@
from .base import Base
class SystemMember(Base):
@classmethod
def ext(cls):
return 'obj'
def system(self):
p = self.symbol.split('-')
return f'{p[0]}-{p[1]}'

13
nullptr/models/waypoint.py
View File

@@ -1,17 +1,14 @@
from .base import Base, Reference
from nullptr.models.system import System
from .system_member import SystemMember
from nullptr.util import *
from time import time
from dataclasses import field
class Waypoint(Base):
class Waypoint(SystemMember):
def define(self):
self.x:int = 0
self.y:int = 0
self.type:str = 'unknown'
self.traits:list = []
self.faction:str = ''
self.system = self.get_system()
self.uncharted = True
def update(self, d):
self.seta('x', d)
@@ -19,9 +16,11 @@ class Waypoint(Base):
self.seta('type', d)
self.seta('faction', d, 'faction.symbol')
self.setlst('traits', d, 'traits', 'symbol')
self.uncharted = 'UNCHARTED' in self.traits
@classmethod
def ext(self):
return 'way'
def path(self):
sector, system, _ = self.symbol.split('-')
return f'atlas/{sector}/{system[0:1]}/{system}/{self.symbol}.{self.ext()}'

236
nullptr/store.py
View File

@@ -1,74 +1,30 @@
from nullptr.models import *
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.system_member import SystemMember
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
from os.path import isfile, dirname, isdir
import os
from os.path import basename
import json
from .util import *
from time import time
import pickle
from struct import unpack, pack
from functools import partial
from io import BytesIO
class StorePickler(pickle.Pickler):
def persistent_id(self, obj):
return "STORE" if type(obj) == Store else None
class StoreUnpickler(pickle.Unpickler):
def __init__(self, stream, store):
self.store = store
super().__init__(stream)
def persistent_load(self, pers_id):
if pers_id == "STORE":
return self.store
raise pickle.UnpicklingError("I don know the persid!")
class ChunkHeader:
def __init__(self):
self.offset = 0
self.in_use = True
self.size = 0
self.used = 0
@classmethod
def parse(cls, fil):
offset = fil.tell()
d = fil.read(16)
if len(d) < 16:
return None
o = cls()
o.offset = offset
d, o.used = unpack('<QQ', d)
o.size = d & 0x7fffffffffffffff
o.in_use = d & 0x8000000000000000 != 0
# print(o)
return o
def write(self, f):
d = self.size
if self.in_use:
d |= 1 << 63
d = pack('<QQ', d, self.used)
f.write(d)
def __repr__(self):
return f'chunk {self.in_use} {self.size} {self.used}'
class Store:
def __init__(self, data_file):
def __init__(self, data_dir):
self.init_models()
self.fil = open_file(data_file)
self.data_dir = data_dir
self.data = {m: {} for m in self.models}
self.system_members = {}
self.dirty_objects = set()
self.cleanup_interval = 600
self.last_cleanup = 0
self.slack = 0.1
self.slack_min = 64
self.slack_max = 1024
self.load()
def init_models(self):
self.models = all_subclasses(Base)
@@ -78,112 +34,53 @@ class Store:
def dirty(self, obj):
self.dirty_objects.add(obj)
def dump_object(self, obj):
buf = BytesIO()
p = StorePickler(buf)
p.dump(obj)
return buf.getvalue()
def path(self, obj):
return os.path.join(self.data_dir, obj.path())
def load_object(self, data, offset):
buf = BytesIO(data)
p = StoreUnpickler(buf, self)
obj = p.load()
obj.file_offset = offset
obj.disable_dirty = False
self.hold(obj)
def load_file(self, path):
if not isfile(path):
return None
fn = basename(path)
ext = fn.split('.')[-1]
symbol = fn.split('.')[0]
if ext not in self.extensions:
return None
with open(path) as f:
data = json.load(f)
typ = self.extensions[ext]
obj = self.create(typ, symbol)
obj.load(data)
obj.store = self
return obj
def load(self):
cnt = 0
start_time = time()
total = 0
free = 0
self.fil.seek(0)
offset = 0
while (hdr := ChunkHeader.parse(self.fil)):
# print(hdr)
total += hdr.size
if not hdr.in_use:
self.fil.seek(hdr.size, 1)
free += hdr.size
continue
data = self.fil.read(hdr.used)
self.load_object(data, offset)
self.fil.seek(hdr.size - hdr.used, 1)
offset = self.fil.tell()
for fil in list_files(self.data_dir, True):
self.load_file(fil)
cnt += 1
dur = time() - start_time
print(f'Loaded {cnt} objects in {dur:.2f} seconds')
print(f'Fragmented space: {free} / {total} bytes')
def allocate_chunk(self, sz):
used = sz
slack = sz * self.slack
slack = min(slack, self.slack_max)
slack = max(slack, self.slack_min)
sz += int(slack)
self.fil.seek(0, 2)
offset = self.fil.tell()
h = ChunkHeader()
h.size = sz
h.used = used
h.offset = self.fil.tell()
h.write(self.fil)
return offset, h
def purge(self, obj):
if obj.file_offset is None:
return
self.fil.seek(obj.file_offset)
hdr = ChunkHeader.parse(self.fil)
hdr.in_use = False
self.fil.seek(obj.file_offset)
hdr.write(self.fil)
obj.file_offset = None
print(f'loaded {cnt} objects in {dur:.2f} seconds')
def store(self, obj):
data = self.dump_object(obj)
osize = len(data)
# is there an existing chunk for this obj?
if obj.file_offset is not None:
# read chunk hdr
self.fil.seek(obj.file_offset)
hdr = ChunkHeader.parse(self.fil)
csize = hdr.size
# if the chunk is too small
if csize < osize:
# free the chunk
hdr.in_use = False
# force a new chunk
obj.file_offset = None
else:
# if it is big enough, update the used field
hdr.used = osize
self.fil.seek(hdr.offset)
hdr.write(self.fil)
if obj.file_offset is None:
obj.file_offset, hdr = self.allocate_chunk(osize)
# print(type(obj).__name__, hdr)
self.fil.write(data)
slack = b'\x00' * (hdr.size - hdr.used)
self.fil.write(slack)
def hold(self, obj):
typ = type(obj)
symbol = obj.symbol
obj.store = self
self.data[typ][symbol] = obj
if hasattr(obj, 'system') and obj.system != None:
system_str = obj.system.symbol
if system_str not in self.system_members:
self.system_members[system_str] = set()
self.system_members[system_str].add(obj)
path = self.path(obj)
path_dir = dirname(path)
data = obj.dict()
if not isdir(path_dir):
os.makedirs(path_dir, exist_ok=True)
with open(path, 'w') as f:
json.dump(data, f, indent=2)
def create(self, typ, symbol):
obj = typ(symbol, self)
self.hold(obj)
self.dirty(obj)
self.data[typ][symbol] = obj
if issubclass(typ, SystemMember):
system_str = obj.system()
if system_str not in self.system_members:
self.system_members[system_str] = set()
self.system_members[system_str].add(obj)
return obj
def get(self, typ, symbol, create=False):
@@ -199,11 +96,6 @@ class Store:
return None
return self.data[typ][symbol]
def getter(self, typ, create=False):
if type(typ) == str and typ in self.model_names:
typ = self.model_names[typ]
return partial(self.get, typ, create=create)
def update(self, typ, data, symbol=None):
if type(typ) == str and typ in self.model_names:
typ = self.model_names[typ]
@@ -221,9 +113,6 @@ class Store:
typ = self.model_names[typ]
for m in self.data[typ].values():
if m.is_expired():
self.dirty(m)
continue
yield m
def all_members(self, system, typ=None):
@@ -235,19 +124,11 @@ class Store:
if system not in self.system_members:
return
garbage = set()
print('typ', typ)
for m in self.system_members[system]:
if m.is_expired():
self.dirty(m)
garbage.add(m)
continue
if typ is None or type(m) == typ:
yield m
for m in garbage:
self.system_members[system].remove(m)
def cleanup(self):
if time() < self.last_cleanup + self.cleanup_interval:
return
@@ -258,8 +139,9 @@ class Store:
if o.is_expired():
expired.append(o)
for o in expired:
self.purge(obj)
path = o.path()
if isfile(path):
os.remove(path)
del self.data[type(o)][o.symbol]
dur = time() - start_time
# print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@@ -270,21 +152,7 @@ class Store:
start_time = time()
for obj in self.dirty_objects:
it += 1
if obj.is_expired():
self.purge(obj)
else:
self.store(obj)
self.fil.flush()
self.dirty_objects = set()
dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}')
def defrag(self):
nm = self.fil.name
self.fil.close()
os.rename(nm, nm + '.bak')
self.fil = open(nm, 'ab+')
for t in self.data:
for o in self.all(t):
o.file_offset = None
self.store(o)

21
nullptr/util.py
View File

@@ -1,15 +1,20 @@
from datetime import datetime
from math import ceil
import os
from os.path import isfile, dirname
from os.path import isfile
def open_file(fn):
d = dirname(fn)
os.makedirs(d, exist_ok=True)
if isfile(fn):
return open(fn, 'rb+')
def list_files(path, recursive=False):
if recursive:
for p, dirnames, fils in os.walk(path):
for f in fils:
fil = os.path.join(p, f)
yield fil
else:
return open(fn, 'ab+')
for f in os.listdir(path):
fil = os.path.join(path, f)
if not isfile(fil):
continue
yield fil
def must_get(d, k):
if type(k) == str:
@@ -53,7 +58,7 @@ def pretty(d, ident=0, detail=2):
return d.f(detail)
r = ''
idt = ' ' * ident
if type(d) in [list, set]:
if type(d) == list:
r += 'lst'
for i in d:
r += '\n' + idt + pretty(i, ident + 1, detail)

50
store.md
View File

@@ -1,50 +0,0 @@
# The store format
This project uses a custom database format just because we can.
The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
The store disk format is optimized for two things:
* Loading the entire contents in memory
* Writing changed objects back to disk
## Objects
First lets discuss what we are storing.
Objects are identified by type and symbol. A symbol is an uppercase alphanumeric string that optionally contains '-' chars. the type is a lowecase alphanumeric string of length 3.
For each type, the store has a class defined that is used to load objects of that type.
An object identifier is its symbol and type joined with a '.' character.
Some examples of object identifiers:
* X1-J84.sys
* X1-J84-0828772.way
* CAPT-J-1.shp
A waypoint is always part of a system. This is also visible because the waypoint symbol is prefixed by the system symbol. However, this relation is not enforced or used by the store. The symbol is an opaque string.
An object has attributes. Values of attributes can be strings, ints, floats, bools, lists, dicts and references. lists and dicts can also only contain the values listed. References are pointers to other objects in the store.
## Indices
An index is a dict with a string as key and a list of objects as value. The dict is built when loading the store. when the index is iterated, each object is re-checked and removed if necessary.
## API
* store.load(fil) loads all objects
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
* store.all(type) generator for all objects of a goven type
* store.purge(obj)
* store.clean() removes all expired objects
* store.flush() writes all dirty objects to disk
* store.defrag() consolidates the store file to minimize storage and loading time
type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
# file format
Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.