richard/0ptr
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New store setup

Browse Source
This commit is contained in:
Richard Bronkhorst 2023-07-10 19:25:01 +02:00
parent 6537db3c03
commit b1e3621490
14 changed files with 212 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -9,4 +9,4 @@ ADD --chown=user . /app
RUN chmod +x /app/main.py
VOLUME /data
ENTRYPOINT [ "python3", "/app/main.py"]
CMD ["-s", "/data/"]
CMD ["-s", "/data/store.npt"]

6
main.py
View File

@ -1,8 +1,8 @@
import argparse
from nullptr.commander import Commander
from nullptr.models.base import Base
def main(args):
c = Commander(args.store_dir)
c = Commander(args.store_file)
c.run()
# X1-AG74-41076A
@ -10,6 +10,6 @@ def main(args):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--store-dir', default='data')
parser.add_argument('-s', '--store-file', default='data/store.npt')
args = parser.parse_args()
main(args)

6
nullptr/api.py
View File

@ -69,6 +69,7 @@ class Api:
}
result = self.request('post', 'register', data, need_token=False)
token = mg(result, 'token')
self.agent.update(mg(result, 'agent'))
self.agent.token = token
def info(self):
@ -87,13 +88,12 @@ class Api:
return self.store.update_list(Waypoint, data)
def marketplace(self, waypoint):
system = waypoint.system()
symbol = str(waypoint)
system = waypoint.system
data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
return self.store.update(Marketplace, data)
def jumps(self, waypoint):
data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate')
data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
symbol = str(waypoint)
return self.store.update(Jumpgate, data, symbol)

47
nullptr/commander.py
View File

@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine
from nullptr.store import Store
from nullptr.analyzer import Analyzer
import argparse
from nullptr.models.agent import Agent
from nullptr.models.system import System
from nullptr.models.waypoint import Waypoint
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models import *
from nullptr.api import Api
from .util import *
from time import sleep, time
@ -17,10 +13,8 @@ class CommandError(Exception):
pass
class Commander(CommandLine):
def __init__(self, store_dir='data'):
self.store_dir = store_dir
self.store = Store(store_dir)
self.store.load()
def __init__(self, store_file='data/store.npt'):
self.store = Store(store_file)
self.agent = self.select_agent()
self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api)
@ -58,6 +52,10 @@ class Commander(CommandLine):
if agent is None:
symbol = input('agent name: ')
agent = self.store.get(Agent, symbol, create=True)
api = Api(self.store, agent)
faction = input('faction: ')
api.register(faction.upper().strip())
self.store.flush()
return agent
def resolve(self, typ, arg):
@ -107,7 +105,7 @@ class Commander(CommandLine):
def do_cmine(self):
if not self.has_ship(): return
site = self.ship.location_str
site = self.ship.location
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@ -118,7 +116,7 @@ class Commander(CommandLine):
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission()
def do_chaul(self):
@ -140,10 +138,10 @@ class Commander(CommandLine):
_, m, _, _ = m[0]
site = self.store.get(Waypoint, m.symbol)
self.centcom.init_mission(self.ship, 'haul')
self.centcom.set_mission_param(self.ship, 'site', site.symbol)
self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol)
self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission()
def do_cprobe(self):
@ -160,15 +158,14 @@ class Commander(CommandLine):
return
markets = [ mkt[1] for mkt in m]
markets = self.analyzer.solve_tsp(markets)
hops = ','.join([m.symbol for m in markets])
self.centcom.init_mission(self.ship, 'probe')
self.centcom.set_mission_param(self.ship, 'hops', hops)
self.centcom.set_mission_param(self.ship, 'hops', markets)
self.print_mission()
def do_travel(self, dest):
dest = self.resolve('Waypoint', dest)
self.centcom.init_mission(self.ship, 'travel')
self.centcom.set_mission_param(self.ship, 'dest', dest.symbol)
self.centcom.set_mission_param(self.ship, 'dest', dest)
self.print_mission()
def do_register(self, faction):
@ -194,7 +191,7 @@ class Commander(CommandLine):
def do_waypoints(self, system_str=''):
if system_str == '':
if not self.has_ship(): return
system = self.ship.location().system()
system = self.ship.location.system
else:
system = self.store.get(System, system_str)
r = self.store.all_members(system, 'Waypoint')
@ -220,7 +217,7 @@ class Commander(CommandLine):
def do_jumps(self, waypoint_str=None):
if waypoint_str is None:
if not self.has_ship(): return
waypoint = self.ship.location()
waypoint = self.ship.location
else:
waypoint = self.store.get(Waypoint, waypoint_str.upper())
r = self.api.jumps(waypoint)
@ -228,7 +225,7 @@ class Commander(CommandLine):
def do_query(self, resource):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
resource = resource.upper()
print('Found markets:')
for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@ -261,7 +258,7 @@ class Commander(CommandLine):
def do_deliver(self):
if not self.has_ship(): return
site = self.ship.location_str
site = self.ship.location
contract = self.active_contract()
delivery = contract.unfinished_delivery()
if delivery is None:
@ -290,7 +287,7 @@ class Commander(CommandLine):
def do_go(self, arg):
if not self.has_ship(): return
system = self.ship.location().system()
system = self.ship.location.system
symbol = f'{system}-{arg}'
dest = self.resolve('Waypoint', symbol)
self.api.navigate(self.ship, dest)
@ -324,7 +321,7 @@ class Commander(CommandLine):
def do_market(self, arg=''):
if arg == '':
if not self.has_ship(): return
waypoint = self.ship.location()
waypoint = self.ship.location
else:
waypoint = self.resolve('Waypoint', arg)
r = self.api.marketplace(waypoint)
@ -354,7 +351,7 @@ class Commander(CommandLine):
def do_shipyard(self):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
data = self.api.shipyard(location)
for s in must_get(data, 'ships'):
print(s['type'], s['purchasePrice'])
@ -362,7 +359,7 @@ class Commander(CommandLine):
def do_jump(self, system_str):
if not self.has_ship(): return
if '-' not in system_str:
sector = self.ship.location_str.split('-')[0]
sector = self.ship.location.system.sector.symbol
system_str = f'{sector}-{system_str}'
system = self.resolve('System', system_str)
self.api.jump(self.ship, system)
@ -370,7 +367,7 @@ class Commander(CommandLine):
def do_purchase(self, ship_type):
if not self.has_ship(): return
location = self.ship.location()
location = self.ship.location
ship_type = ship_type.upper()
if not ship_type.startswith('SHIP'):
ship_type = 'SHIP_' + ship_type

12
nullptr/models/__init__.py
View File

@ -0,0 +1,12 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

47
nullptr/models/base.py
View File

@ -1,18 +1,38 @@
from copy import deepcopy
from nullptr.util import sg
class Reference:
def __init__(self, typ, symbol, store):
self.typ = typ
self.symbol = symbol
self.store = store
@classmethod
def create(cls, obj):
o = cls(type(obj), obj.symbol, obj.store)
return o
def resolve(self):
self.store.get(self.typ, self.symbol)
def __repr__(self):
return f'*REF*{self.symbol}.{self.typ.ext()}'
class Base:
identifier = 'symbol'
symbol: str
store: object
def __init__(self, symbol, store):
self.disable_dirty = True
self.file_offset = 0
self.store = store
self.symbol = symbol
self.define()
self.disable_dirty = False
@classmethod
def ext(cls):
raise NotImplementedError('no ext')
def define(self):
pass
@ -38,10 +58,18 @@ class Base:
setattr(self, attr, lst)
def __setattr__(self, name, value):
if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty:
if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
self.store.dirty(self)
if issubclass(type(value), Base):
value = Reference.create(value)
super().__setattr__(name, value)
def __getattribute__(self, nm):
val = super().__getattribute__(nm)
if type(val) == Reference:
val = val.resolve()
return val
def update(self, d):
pass
@ -52,22 +80,15 @@ class Base:
self.disable_dirty = True
self.__dict__.update(d)
self.disable_dirty = False
def dict(self):
def __getstate__(self):
r = {}
for k,v in self.__dict__.items():
if k in ['store']:
if k in ['store','file_offset', 'disable_dirty', 'file_offset']:
continue
r[k] = deepcopy(v)
return r
def path(self):
raise NotImplementedError('path')
@classmethod
def ext(self):
raise NotImplementedError('extension')
def type(self):
return self.__class__.__name__

4
nullptr/models/jumpgate.py
View File

@ -1,7 +1,7 @@
from .system_member import SystemMember
from .base import Base
from dataclasses import field
class Jumpgate(SystemMember):
class Jumpgate(Base):
def define(self):
self.range: int = 0
self.faction: str = ''

4
nullptr/models/marketplace.py
View File

@ -1,10 +1,10 @@
from .system_member import SystemMember
from .base import Base
from time import time
from nullptr.util import *
from dataclasses import field
class Marketplace(SystemMember):
class Marketplace(Base):
def define(self):
self.imports:list = []
self.exports:list = []

4
nullptr/models/survey.py
View File

@ -1,10 +1,10 @@
from time import time
from nullptr.util import *
from .system_member import SystemMember
from .base import Base
size_names = ['SMALL','MODERATE','LARGE']
class Survey(SystemMember):
class Survey(Base):
identifier = 'signature'
def define(self):
self.type: str = ''

10
nullptr/models/system_member.py
View File

@ -1,10 +0,0 @@
from .base import Base
class SystemMember(Base):
@classmethod
def ext(cls):
return 'obj'
def system(self):
p = self.symbol.split('-')
return f'{p[0]}-{p[1]}'

4
nullptr/models/waypoint.py
View File

@ -1,8 +1,8 @@
from .system_member import SystemMember
from .base import Base
from nullptr.util import *
from dataclasses import field
class Waypoint(SystemMember):
class Waypoint(Base):
def define(self):
self.x:int = 0
self.y:int = 0

165
nullptr/store.py
View File

@ -1,30 +1,55 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.system_member import SystemMember
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
from nullptr.models import *
from os.path import isfile, dirname, isdir
import os
from os.path import basename
import json
from .util import *
from time import time
import pickle
from struct import unpack, pack
class ChunkHeader:
def __init__(self):
self.in_use = True
self.size = 0
self.used = 0
@classmethod
def parse(cls, fil):
d = fil.read(16)
if len(d) < 16:
return None
# print(d)
o = cls()
d, o.used = unpack('<QQ', d)
o.size = d & 0x7fffffffffffffff
o.in_use = d & 0x8000000000000000 != 0
# print(o)
return o
def write(self, f):
d = self.size
if self.in_use:
d |= 1 << 63
d = pack('<QQ', d, self.used)
f.write(d)
def __repr__(self):
return f'chunk {self.in_use} {self.size} {self.used}'
class Store:
def __init__(self, data_dir):
def __init__(self, data_file):
self.init_models()
self.data_dir = data_dir
self.fil = open_file(data_file)
self.data = {m: {} for m in self.models}
self.system_members = {}
self.dirty_objects = set()
self.cleanup_interval = 600
self.last_cleanup = 0
self.slack = 0.1
self.slack_min = 64
self.slack_max = 1024
self.load()
def init_models(self):
self.models = all_subclasses(Base)
@ -33,54 +58,89 @@ class Store:
def dirty(self, obj):
self.dirty_objects.add(obj)
def path(self, obj):
return os.path.join(self.data_dir, obj.path())
def load_file(self, path):
if not isfile(path):
return None
fn = basename(path)
ext = fn.split('.')[-1]
symbol = fn.split('.')[0]
if ext not in self.extensions:
return None
with open(path) as f:
data = json.load(f)
typ = self.extensions[ext]
obj = self.create(typ, symbol)
obj.load(data)
obj.store = self
return obj
def dump_object(self, obj):
return pickle.dumps(obj)
def load_object(self, data, offset):
obj = pickle.loads(data)
obj.file_offset = offset
obj.disable_dirty = False
self.hold(obj)
def load(self):
cnt = 0
start_time = time()
for fil in list_files(self.data_dir, True):
self.load_file(fil)
self.fil.seek(0)
offset = 0
while (hdr := ChunkHeader.parse(self.fil)):
if not hdr.in_use: continue
data = self.fil.read(hdr.used)
self.load_object(data, offset)
self.fil.seek(hdr.size - hdr.used, 1)
offset = self.fil.tell()
cnt += 1
dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds')
def allocate_chunk(self, sz):
used = sz
slack = sz * self.slack
slack = min(slack, self.slack_max)
slack = max(slack, self.slack_min)
sz += slack
self.fil.seek(0, 2)
offset = self.fil.tell()
h = ChunkHeader()
h.size = sz
h.used = used
h.write(self.fil)
return offset, h
def store(self, obj):
path = self.path(obj)
path_dir = dirname(path)
data = obj.dict()
if not isdir(path_dir):
os.makedirs(path_dir, exist_ok=True)
with open(path, 'w') as f:
json.dump(data, f, indent=2)
def create(self, typ, symbol):
obj = typ(symbol, self)
data = self.dump_object(obj)
osize = len(data)
# is there an existing chunk for this obj?
if obj.file_offset > 0:
# read chunk hdr
self.fil.seek(obj.file_offset)
hdr = ChunkHeader.parse(self.fil)
csize = hdr.size
# if the chunk is too small
if csize < osize:
# free the chunk
hdr.in_use = False
# force a new chunk
obj.file_offset = 0
else:
# if it is big enough, update the used field
hdr.used = osize
self.fil.seek(obj.file_offset)
hdr.write(self.fil)
if obj.file_offset == 0:
obj.file_offset, hdr = self.allocate_chunk(osize)
self.fil.write(data)
slack = b'\x00' * (hdr.size - hdr.used)
self.fil.write(slack)
def hold(self, obj):
typ = type(obj)
symbol = obj.symbol
obj.store = self
self.data[typ][symbol] = obj
if issubclass(typ, SystemMember):
system_str = obj.system()
if hasattr(typ, 'system'):
system_str = obj.system.symbol
if system_str not in self.system_members:
self.system_members[system_str] = set()
self.system_members[system_str].add(obj)
def create(self, typ, symbol):
obj = typ(symbol, self)
self.hold(obj)
self.dirty(obj)
return obj
def get(self, typ, symbol, create=False):
@ -122,7 +182,7 @@ class Store:
if type(system) == System:
system = system.symbol
if system not in self.system_members:
if 'system' not in self.system_members:
return
print('typ', typ)
for m in self.system_members[system]:
@ -139,9 +199,9 @@ class Store:
if o.is_expired():
expired.append(o)
for o in expired:
path = o.path()
if isfile(path):
os.remove(path)
# TODO
del self.data[type(o)][o.symbol]
dur = time() - start_time
# print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@ -153,6 +213,7 @@ class Store:
for obj in self.dirty_objects:
it += 1
self.store(obj)
self.fil.flush()
self.dirty_objects = set()
dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}')

21
nullptr/util.py
View File

@ -1,21 +1,16 @@
from datetime import datetime
from math import ceil
import os
from os.path import isfile
from os.path import isfile, dirname
def list_files(path, recursive=False):
if recursive:
for p, dirnames, fils in os.walk(path):
for f in fils:
fil = os.path.join(p, f)
yield fil
def open_file(fn):
d = dirname(fn)
os.makedirs(d, exist_ok=True)
if isfile(fn):
return open(fn, 'rb+')
else:
for f in os.listdir(path):
fil = os.path.join(path, f)
if not isfile(fil):
continue
yield fil
return open(fn, 'ab+')
def must_get(d, k):
if type(k) == str:
k = k.split('.')

16
store.md
View File

@ -3,6 +3,10 @@ This project uses a custom database format just because we can.
The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
The store disk format is optimized for two things:
* Loading the entire contents in memory
* Writing changed objects back to disk
## Objects
First lets discuss what we are storing.
@ -30,21 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict
* store.load(fil) loads all objects
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
* store.all(type) generator for all objects of a goven type
* store.delete(typ, symbol)
* store.cleanup() removes all expired objects
* store.flush() writes all dirty objects to disk
*
* store.defrag() consolidates the store file to minimize storage and loading time
type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
# file format
Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header:
The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
* Magic
* Blocksize in bytes
* Number of blocks
* Root file
* Free file
A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.
A block is prefixed by a pointer to the next block of that file. In the last block of the file, the pointer is 0. So if you have 1000 byte blocks, each block takes 1008 bytes of space.