diff --git a/Dockerfile b/Dockerfile index 90091d9..4f04926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,4 +9,4 @@ ADD --chown=user . /app RUN chmod +x /app/main.py VOLUME /data ENTRYPOINT [ "python3", "/app/main.py"] -CMD ["-s", "/data/"] +CMD ["-s", "/data/store.npt"] diff --git a/main.py b/main.py index 013be4b..15b2b75 100644 --- a/main.py +++ b/main.py @@ -1,8 +1,8 @@ import argparse from nullptr.commander import Commander - +from nullptr.models.base import Base def main(args): - c = Commander(args.store_dir) + c = Commander(args.store_file) c.run() # X1-AG74-41076A @@ -10,6 +10,6 @@ def main(args): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('-s', '--store-dir', default='data') + parser.add_argument('-s', '--store-file', default='data/store.npt') args = parser.parse_args() main(args) diff --git a/nullptr/api.py b/nullptr/api.py index 0c83211..644b957 100644 --- a/nullptr/api.py +++ b/nullptr/api.py @@ -69,6 +69,7 @@ class Api: } result = self.request('post', 'register', data, need_token=False) token = mg(result, 'token') + self.agent.update(mg(result, 'agent')) self.agent.token = token def info(self): @@ -87,13 +88,12 @@ class Api: return self.store.update_list(Waypoint, data) def marketplace(self, waypoint): - system = waypoint.system() - symbol = str(waypoint) + system = waypoint.system data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market') return self.store.update(Marketplace, data) def jumps(self, waypoint): - data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate') + data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate') symbol = str(waypoint) return self.store.update(Jumpgate, data, symbol) diff --git a/nullptr/commander.py b/nullptr/commander.py index 1a223f6..7b12842 100644 --- a/nullptr/commander.py +++ b/nullptr/commander.py @@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine from nullptr.store import Store from nullptr.analyzer import Analyzer import argparse -from nullptr.models.agent import Agent -from nullptr.models.system import System -from nullptr.models.waypoint import Waypoint -from nullptr.models.marketplace import Marketplace -from nullptr.models.jumpgate import Jumpgate +from nullptr.models import * from nullptr.api import Api from .util import * from time import sleep, time @@ -17,10 +13,8 @@ class CommandError(Exception): pass class Commander(CommandLine): - def __init__(self, store_dir='data'): - self.store_dir = store_dir - self.store = Store(store_dir) - self.store.load() + def __init__(self, store_file='data/store.npt'): + self.store = Store(store_file) self.agent = self.select_agent() self.api = Api(self.store, self.agent) self.atlas_builder = AtlasBuilder(self.store, self.api) @@ -58,6 +52,10 @@ class Commander(CommandLine): if agent is None: symbol = input('agent name: ') agent = self.store.get(Agent, symbol, create=True) + api = Api(self.store, agent) + faction = input('faction: ') + api.register(faction.upper().strip()) + self.store.flush() return agent def resolve(self, typ, arg): @@ -107,7 +105,7 @@ class Commander(CommandLine): def do_cmine(self): if not self.has_ship(): return - site = self.ship.location_str + site = self.ship.location contract = self.active_contract() delivery = contract.unfinished_delivery() if delivery is None: @@ -118,7 +116,7 @@ class Commander(CommandLine): self.centcom.set_mission_param(self.ship, 'site', site) self.centcom.set_mission_param(self.ship, 'resource', resource) self.centcom.set_mission_param(self.ship, 'dest', destination) - self.centcom.set_mission_param(self.ship, 'contract', contract.symbol) + self.centcom.set_mission_param(self.ship, 'contract', contract) self.print_mission() def do_chaul(self): @@ -140,10 +138,10 @@ class Commander(CommandLine): _, m, _, _ = m[0] site = self.store.get(Waypoint, m.symbol) self.centcom.init_mission(self.ship, 'haul') - self.centcom.set_mission_param(self.ship, 'site', site.symbol) + self.centcom.set_mission_param(self.ship, 'site', site) self.centcom.set_mission_param(self.ship, 'resource', resource) self.centcom.set_mission_param(self.ship, 'dest', destination) - self.centcom.set_mission_param(self.ship, 'contract', contract.symbol) + self.centcom.set_mission_param(self.ship, 'contract', contract) self.print_mission() def do_cprobe(self): @@ -160,15 +158,14 @@ class Commander(CommandLine): return markets = [ mkt[1] for mkt in m] markets = self.analyzer.solve_tsp(markets) - hops = ','.join([m.symbol for m in markets]) self.centcom.init_mission(self.ship, 'probe') - self.centcom.set_mission_param(self.ship, 'hops', hops) + self.centcom.set_mission_param(self.ship, 'hops', markets) self.print_mission() def do_travel(self, dest): dest = self.resolve('Waypoint', dest) self.centcom.init_mission(self.ship, 'travel') - self.centcom.set_mission_param(self.ship, 'dest', dest.symbol) + self.centcom.set_mission_param(self.ship, 'dest', dest) self.print_mission() def do_register(self, faction): @@ -194,7 +191,7 @@ class Commander(CommandLine): def do_waypoints(self, system_str=''): if system_str == '': if not self.has_ship(): return - system = self.ship.location().system() + system = self.ship.location.system else: system = self.store.get(System, system_str) r = self.store.all_members(system, 'Waypoint') @@ -220,7 +217,7 @@ class Commander(CommandLine): def do_jumps(self, waypoint_str=None): if waypoint_str is None: if not self.has_ship(): return - waypoint = self.ship.location() + waypoint = self.ship.location else: waypoint = self.store.get(Waypoint, waypoint_str.upper()) r = self.api.jumps(waypoint) @@ -228,7 +225,7 @@ class Commander(CommandLine): def do_query(self, resource): if not self.has_ship(): return - location = self.ship.location() + location = self.ship.location resource = resource.upper() print('Found markets:') for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location): @@ -261,7 +258,7 @@ class Commander(CommandLine): def do_deliver(self): if not self.has_ship(): return - site = self.ship.location_str + site = self.ship.location contract = self.active_contract() delivery = contract.unfinished_delivery() if delivery is None: @@ -290,7 +287,7 @@ class Commander(CommandLine): def do_go(self, arg): if not self.has_ship(): return - system = self.ship.location().system() + system = self.ship.location.system symbol = f'{system}-{arg}' dest = self.resolve('Waypoint', symbol) self.api.navigate(self.ship, dest) @@ -324,7 +321,7 @@ class Commander(CommandLine): def do_market(self, arg=''): if arg == '': if not self.has_ship(): return - waypoint = self.ship.location() + waypoint = self.ship.location else: waypoint = self.resolve('Waypoint', arg) r = self.api.marketplace(waypoint) @@ -354,7 +351,7 @@ class Commander(CommandLine): def do_shipyard(self): if not self.has_ship(): return - location = self.ship.location() + location = self.ship.location data = self.api.shipyard(location) for s in must_get(data, 'ships'): print(s['type'], s['purchasePrice']) @@ -362,7 +359,7 @@ class Commander(CommandLine): def do_jump(self, system_str): if not self.has_ship(): return if '-' not in system_str: - sector = self.ship.location_str.split('-')[0] + sector = self.ship.location.system.sector.symbol system_str = f'{sector}-{system_str}' system = self.resolve('System', system_str) self.api.jump(self.ship, system) @@ -370,7 +367,7 @@ class Commander(CommandLine): def do_purchase(self, ship_type): if not self.has_ship(): return - location = self.ship.location() + location = self.ship.location ship_type = ship_type.upper() if not ship_type.startswith('SHIP'): ship_type = 'SHIP_' + ship_type diff --git a/nullptr/models/__init__.py b/nullptr/models/__init__.py index e69de29..f919fce 100644 --- a/nullptr/models/__init__.py +++ b/nullptr/models/__init__.py @@ -0,0 +1,12 @@ +from nullptr.models.base import Base +from nullptr.models.waypoint import Waypoint +from nullptr.models.sector import Sector +from nullptr.models.system import System +from nullptr.models.agent import Agent +from nullptr.models.marketplace import Marketplace +from nullptr.models.jumpgate import Jumpgate +from nullptr.models.ship import Ship +from nullptr.models.contract import Contract +from nullptr.models.survey import Survey + +__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ] diff --git a/nullptr/models/base.py b/nullptr/models/base.py index 5f80271..5d359b3 100644 --- a/nullptr/models/base.py +++ b/nullptr/models/base.py @@ -1,18 +1,38 @@ from copy import deepcopy from nullptr.util import sg +class Reference: + def __init__(self, typ, symbol, store): + self.typ = typ + self.symbol = symbol + self.store = store + + @classmethod + def create(cls, obj): + o = cls(type(obj), obj.symbol, obj.store) + return o + + def resolve(self): + self.store.get(self.typ, self.symbol) + + def __repr__(self): + return f'*REF*{self.symbol}.{self.typ.ext()}' + class Base: identifier = 'symbol' - symbol: str - store: object def __init__(self, symbol, store): self.disable_dirty = True + self.file_offset = 0 self.store = store self.symbol = symbol self.define() self.disable_dirty = False + @classmethod + def ext(cls): + raise NotImplementedError('no ext') + def define(self): pass @@ -38,10 +58,18 @@ class Base: setattr(self, attr, lst) def __setattr__(self, name, value): - if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty: + if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty: self.store.dirty(self) + if issubclass(type(value), Base): + value = Reference.create(value) super().__setattr__(name, value) + def __getattribute__(self, nm): + val = super().__getattribute__(nm) + if type(val) == Reference: + val = val.resolve() + return val + def update(self, d): pass @@ -52,22 +80,15 @@ class Base: self.disable_dirty = True self.__dict__.update(d) self.disable_dirty = False - - def dict(self): + + def __getstate__(self): r = {} for k,v in self.__dict__.items(): - if k in ['store']: + if k in ['store','file_offset', 'disable_dirty', 'file_offset']: continue r[k] = deepcopy(v) return r - def path(self): - raise NotImplementedError('path') - - @classmethod - def ext(self): - raise NotImplementedError('extension') - def type(self): return self.__class__.__name__ diff --git a/nullptr/models/jumpgate.py b/nullptr/models/jumpgate.py index 068b7f6..72ae7d0 100644 --- a/nullptr/models/jumpgate.py +++ b/nullptr/models/jumpgate.py @@ -1,7 +1,7 @@ -from .system_member import SystemMember +from .base import Base from dataclasses import field -class Jumpgate(SystemMember): +class Jumpgate(Base): def define(self): self.range: int = 0 self.faction: str = '' diff --git a/nullptr/models/marketplace.py b/nullptr/models/marketplace.py index ca6e4ae..5187cdf 100644 --- a/nullptr/models/marketplace.py +++ b/nullptr/models/marketplace.py @@ -1,10 +1,10 @@ -from .system_member import SystemMember +from .base import Base from time import time from nullptr.util import * from dataclasses import field -class Marketplace(SystemMember): +class Marketplace(Base): def define(self): self.imports:list = [] self.exports:list = [] diff --git a/nullptr/models/survey.py b/nullptr/models/survey.py index c673017..7ea3dd8 100644 --- a/nullptr/models/survey.py +++ b/nullptr/models/survey.py @@ -1,10 +1,10 @@ from time import time from nullptr.util import * -from .system_member import SystemMember +from .base import Base size_names = ['SMALL','MODERATE','LARGE'] -class Survey(SystemMember): +class Survey(Base): identifier = 'signature' def define(self): self.type: str = '' diff --git a/nullptr/models/system_member.py b/nullptr/models/system_member.py deleted file mode 100644 index b031a1d..0000000 --- a/nullptr/models/system_member.py +++ /dev/null @@ -1,10 +0,0 @@ -from .base import Base - -class SystemMember(Base): - @classmethod - def ext(cls): - return 'obj' - - def system(self): - p = self.symbol.split('-') - return f'{p[0]}-{p[1]}' diff --git a/nullptr/models/waypoint.py b/nullptr/models/waypoint.py index 4d5dbb0..8207580 100644 --- a/nullptr/models/waypoint.py +++ b/nullptr/models/waypoint.py @@ -1,8 +1,8 @@ -from .system_member import SystemMember +from .base import Base from nullptr.util import * from dataclasses import field -class Waypoint(SystemMember): +class Waypoint(Base): def define(self): self.x:int = 0 self.y:int = 0 diff --git a/nullptr/store.py b/nullptr/store.py index 91ef91d..66bb784 100644 --- a/nullptr/store.py +++ b/nullptr/store.py @@ -1,30 +1,55 @@ -from nullptr.models.base import Base -from nullptr.models.waypoint import Waypoint -from nullptr.models.sector import Sector -from nullptr.models.system import System -from nullptr.models.agent import Agent -from nullptr.models.marketplace import Marketplace -from nullptr.models.system_member import SystemMember -from nullptr.models.jumpgate import Jumpgate -from nullptr.models.ship import Ship -from nullptr.models.contract import Contract -from nullptr.models.survey import Survey +from nullptr.models import * from os.path import isfile, dirname, isdir import os from os.path import basename import json from .util import * from time import time +import pickle +from struct import unpack, pack +class ChunkHeader: + def __init__(self): + self.in_use = True + self.size = 0 + self.used = 0 + + @classmethod + def parse(cls, fil): + d = fil.read(16) + if len(d) < 16: + return None + # print(d) + o = cls() + d, o.used = unpack(' 0: + # read chunk hdr + self.fil.seek(obj.file_offset) + hdr = ChunkHeader.parse(self.fil) + csize = hdr.size + # if the chunk is too small + if csize < osize: + # free the chunk + hdr.in_use = False + # force a new chunk + obj.file_offset = 0 + else: + # if it is big enough, update the used field + hdr.used = osize + self.fil.seek(obj.file_offset) + hdr.write(self.fil) + + if obj.file_offset == 0: + obj.file_offset, hdr = self.allocate_chunk(osize) + self.fil.write(data) + slack = b'\x00' * (hdr.size - hdr.used) + self.fil.write(slack) + + def hold(self, obj): + typ = type(obj) + symbol = obj.symbol + obj.store = self self.data[typ][symbol] = obj - if issubclass(typ, SystemMember): - system_str = obj.system() - + if hasattr(typ, 'system'): + system_str = obj.system.symbol if system_str not in self.system_members: self.system_members[system_str] = set() self.system_members[system_str].add(obj) + + def create(self, typ, symbol): + obj = typ(symbol, self) + self.hold(obj) + self.dirty(obj) return obj def get(self, typ, symbol, create=False): @@ -122,7 +182,7 @@ class Store: if type(system) == System: system = system.symbol - if system not in self.system_members: + if 'system' not in self.system_members: return print('typ', typ) for m in self.system_members[system]: @@ -139,9 +199,9 @@ class Store: if o.is_expired(): expired.append(o) for o in expired: - path = o.path() - if isfile(path): - os.remove(path) + + # TODO + del self.data[type(o)][o.symbol] dur = time() - start_time # print(f'cleaned {len(expired)} in {dur:.03f} seconds') @@ -153,6 +213,7 @@ class Store: for obj in self.dirty_objects: it += 1 self.store(obj) + self.fil.flush() self.dirty_objects = set() dur = time() - start_time # print(f'flush done {it} items {dur:.2f}') diff --git a/nullptr/util.py b/nullptr/util.py index a5ea388..c2d9157 100644 --- a/nullptr/util.py +++ b/nullptr/util.py @@ -1,21 +1,16 @@ from datetime import datetime from math import ceil import os -from os.path import isfile +from os.path import isfile, dirname -def list_files(path, recursive=False): - if recursive: - for p, dirnames, fils in os.walk(path): - for f in fils: - fil = os.path.join(p, f) - yield fil +def open_file(fn): + d = dirname(fn) + os.makedirs(d, exist_ok=True) + if isfile(fn): + return open(fn, 'rb+') else: - for f in os.listdir(path): - fil = os.path.join(path, f) - if not isfile(fil): - continue - yield fil - + return open(fn, 'ab+') + def must_get(d, k): if type(k) == str: k = k.split('.') diff --git a/store.md b/store.md index 12e65c0..978f340 100644 --- a/store.md +++ b/store.md @@ -3,6 +3,10 @@ This project uses a custom database format just because we can. The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk. +The store disk format is optimized for two things: +* Loading the entire contents in memory +* Writing changed objects back to disk + ## Objects First lets discuss what we are storing. @@ -30,21 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict * store.load(fil) loads all objects * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present * store.all(type) generator for all objects of a goven type +* store.delete(typ, symbol) * store.cleanup() removes all expired objects * store.flush() writes all dirty objects to disk -* +* store.defrag() consolidates the store file to minimize storage and loading time type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base # file format Until specified otherwise, all numbers are stored low-endian 64bit unsigned. -the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header: +The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty. -* Magic -* Blocksize in bytes -* Number of blocks -* Root file -* Free file +A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading. -A block is prefixed by a pointer to the next block of that file. In the last block of the file, the pointer is 0. So if you have 1000 byte blocks, each block takes 1008 bytes of space. \ No newline at end of file