From e5c384caa9eb45c07fcbe8fc920293571eca716c Mon Sep 17 00:00:00 2001 From: Richard Bronkhorst Date: Sun, 16 Jul 2023 18:48:45 +0200 Subject: [PATCH] Rewrite atlas builder to be re-entrant. Rolled into automode. --- nullptr/api.py | 2 +- nullptr/atlas_builder.py | 107 +++++++++++++++++++------------------ nullptr/central_command.py | 5 ++ nullptr/commander.py | 11 ++-- nullptr/models/__init__.py | 3 +- nullptr/models/atlas.py | 11 ++++ nullptr/models/system.py | 2 + nullptr/models/waypoint.py | 5 +- nullptr/store.py | 2 +- 9 files changed, 88 insertions(+), 60 deletions(-) create mode 100644 nullptr/models/atlas.py diff --git a/nullptr/api.py b/nullptr/api.py index 052ace4..1d4cc8c 100644 --- a/nullptr/api.py +++ b/nullptr/api.py @@ -19,7 +19,7 @@ class Api: self.agent = agent self.store = store self.requests_sent = 0 - self.meta = None + self.last_meta = None self.last_result = None self.root = 'https://api.spacetraders.io/v2/' diff --git a/nullptr/atlas_builder.py b/nullptr/atlas_builder.py index 3c15eaf..cce0ff5 100644 --- a/nullptr/atlas_builder.py +++ b/nullptr/atlas_builder.py @@ -1,66 +1,71 @@ -from time import sleep +from time import sleep, time from nullptr.util import * from threading import Thread +from nullptr.models.atlas import Atlas +from functools import partial +from nullptr.models import System class AtlasBuilder: def __init__(self, store, api): self.store = store self.api = api - self.stop_auto = False - - def wait_for_stop(self): - try: - input() - except EOFError: - pass - self.stop_auto = True - print('stopping...') - - def run(self, page=1): - print('universe mode. hit enter to stop') - t = Thread(target=self.wait_for_stop) - t.daemon = True - t.start() - self.all_systems(int(page)) - print('manual mode') + self.work = [] + self.max_work = 100 + self.unch_interval = 86400 + self.atlas = self.store.get(Atlas, 'ATLAS', create=True) - def all_specials(self, waypoints): - for w in waypoints: - if self.stop_auto: + def find_work(self): + first_page = self.atlas.total_pages == 0 + pages_left = self.atlas.total_pages < self.atlas.seen_pages + if first_page or pages_left: + self.sched(self.get_systems) + return + for s in self.store.all(System): + if len(self.work) > self.max_work: break + if not s.uncharted: continue + if s.last_crawl > time() - self.unch_interval: + continue + self.sched(self.get_waypoints, s) + + + def do_work(self): + if len(self.work) == 0: + self.find_work() + if len(self.work) == 0: + return + work = self.work.pop() + work() + + def get_systems(self): + page = 1 + if self.atlas.seen_pages > 0: + page = self.atlas.seen_pages + 1 + if page > self.atlas.total_pages: + return + data = self.api.list_systems(page) + self.atlas.total_pages = total_pages(self.api.last_meta) + self.atlas.seen_pages = page + + def get_waypoints(self, system): + wps = self.api.list_waypoints(system) + system.last_crawl = time() + system.uncharted = len([1 for w in wps if w.uncharted]) > 0 + self.schedule_specials(wps) + + def sched(self, fun, *args): + self.work.append(partial(fun, *args)) + + def schedule_specials(self, waypoints): + for w in waypoints: if 'UNCHARTED' in w.traits: continue if 'MARKETPLACE' in w.traits: print(f'marketplace at {w}') - self.api.marketplace(w) - sleep(0.5) + self.sched(self.api.marketplace, w) if w.type == 'JUMP_GATE': print(f'jumpgate at {w}') - self.api.jumps(w) - - def all_waypoints(self, systems): - for s in systems: - if self.stop_auto: - break - r = self.api.list_waypoints(s) - self.all_specials(r) - sleep(0.5) - - - def all_systems(self, start_page): - self.stop_auto = False - data = self.api.list_systems(start_page) - pages = total_pages(self.api.last_meta) - print(f'{pages} pages of systems') - print(f'page {1}: {len(data)} results') - self.all_waypoints(data) - self.store.flush() - - for p in range(start_page+1, pages+1): - if self.stop_auto: - break - data = self.api.list_systems(p) - print(f'page {p}: {len(data)} systems') - self.all_waypoints(data) - sleep(0.5) - self.store.flush() + self.sched(self.api.jumps, w) + if 'SHIPYARD' in w.traits: + # todo + pass diff --git a/nullptr/central_command.py b/nullptr/central_command.py index 7c6f590..6c2398f 100644 --- a/nullptr/central_command.py +++ b/nullptr/central_command.py @@ -4,6 +4,7 @@ from nullptr.missions import create_mission, get_mission_class from random import choice from time import sleep from threading import Thread +from nullptr.atlas_builder import AtlasBuilder class CentralCommandError(Exception): pass @@ -14,6 +15,7 @@ class CentralCommand: self.stopping = False self.store = store self.api = api + self.atlas_builder = AtlasBuilder(store, api) self.update_missions() def get_ready_missions(self): @@ -56,6 +58,9 @@ class CentralCommand: request_counter = self.api.requests_sent while request_counter == self.api.requests_sent and did_step: did_step = self.tick() + if request_counter == self.api.requests_sent: + self.atlas_builder.do_work() + self.store.flush() sleep(0.5) self.stopping = False diff --git a/nullptr/commander.py b/nullptr/commander.py index d0eeb3f..ca3b870 100644 --- a/nullptr/commander.py +++ b/nullptr/commander.py @@ -7,8 +7,8 @@ from nullptr.api import Api from .util import * from time import sleep, time from threading import Thread -from nullptr.atlas_builder import AtlasBuilder from nullptr.central_command import CentralCommand + class CommandError(Exception): pass @@ -17,7 +17,6 @@ class Commander(CommandLine): self.store = Store(store_file) self.agent = self.select_agent() self.api = Api(self.store, self.agent) - self.atlas_builder = AtlasBuilder(self.store, self.api) self.centcom = CentralCommand(self.store, self.api) self.analyzer = Analyzer(self.store) self.ship = None @@ -183,9 +182,6 @@ class Commander(CommandLine): self.api.register(faction.upper()) pprint(self.api.agent) - def do_universe(self, page=1): - self.atlas_builder.run(page) - def do_systems(self, page=1): r = self.api.list_systems(int(page)) pprint(self.api.last_meta) @@ -201,6 +197,11 @@ class Commander(CommandLine): def do_defrag(self): self.store.defrag() + + def do_system(self, system_str): + system = self.store.get(System, system_str) + r = self.api.list_waypoints(system) + pprint(r) def do_waypoints(self, system_str=''): if system_str == '': diff --git a/nullptr/models/__init__.py b/nullptr/models/__init__.py index c9063dc..d768312 100644 --- a/nullptr/models/__init__.py +++ b/nullptr/models/__init__.py @@ -8,5 +8,6 @@ from nullptr.models.jumpgate import Jumpgate from nullptr.models.ship import Ship from nullptr.models.contract import Contract from nullptr.models.survey import Survey +from nullptr.models.atlas import Atlas -__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ] +__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base', 'Atlas' ] diff --git a/nullptr/models/atlas.py b/nullptr/models/atlas.py new file mode 100644 index 0000000..5851e7c --- /dev/null +++ b/nullptr/models/atlas.py @@ -0,0 +1,11 @@ +from .base import Base + +class Atlas(Base): + @classmethod + def ext(self): + return 'atl' + + def define(self): + self.total_pages = 0 + self.seen_pages = 0 + diff --git a/nullptr/models/system.py b/nullptr/models/system.py index 9ee91ba..41842a4 100644 --- a/nullptr/models/system.py +++ b/nullptr/models/system.py @@ -7,6 +7,8 @@ class System(Base): self.x:int = 0 self.y:int = 0 self.type:str = 'unknown' + self.uncharted = True + self.last_crawl = 0 def update(self, d): self.seta('x', d) diff --git a/nullptr/models/waypoint.py b/nullptr/models/waypoint.py index 4172b5e..24f007b 100644 --- a/nullptr/models/waypoint.py +++ b/nullptr/models/waypoint.py @@ -1,6 +1,7 @@ from .base import Base, Reference from nullptr.models.system import System from nullptr.util import * +from time import time class Waypoint(Base): def define(self): @@ -10,6 +11,7 @@ class Waypoint(Base): self.traits:list = [] self.faction:str = '' self.system = self.get_system() + self.uncharted = True def update(self, d): self.seta('x', d) @@ -17,7 +19,8 @@ class Waypoint(Base): self.seta('type', d) self.seta('faction', d, 'faction.symbol') self.setlst('traits', d, 'traits', 'symbol') - + self.uncharted = 'UNCHARTED' in self.traits + @classmethod def ext(self): return 'way' diff --git a/nullptr/store.py b/nullptr/store.py index 188983f..0235da4 100644 --- a/nullptr/store.py +++ b/nullptr/store.py @@ -113,7 +113,7 @@ class Store: cnt += 1 dur = time() - start_time - print(f'loaded {cnt} objects in {dur:.2f} seconds') + print(f'Loaded {cnt} objects in {dur:.2f} seconds') print(f'Fragmented space: {free} / {total} bytes') def allocate_chunk(self, sz):