Rewrite atlas builder to be re-entrant. Rolled into automode.

This commit is contained in:
Richard Bronkhorst 2023-07-16 18:48:45 +02:00
parent f644027750
commit e5c384caa9
9 changed files with 88 additions and 60 deletions

View File

@ -19,7 +19,7 @@ class Api:
self.agent = agent self.agent = agent
self.store = store self.store = store
self.requests_sent = 0 self.requests_sent = 0
self.meta = None self.last_meta = None
self.last_result = None self.last_result = None
self.root = 'https://api.spacetraders.io/v2/' self.root = 'https://api.spacetraders.io/v2/'

View File

@ -1,66 +1,71 @@
from time import sleep from time import sleep, time
from nullptr.util import * from nullptr.util import *
from threading import Thread from threading import Thread
from nullptr.models.atlas import Atlas
from functools import partial
from nullptr.models import System
class AtlasBuilder: class AtlasBuilder:
def __init__(self, store, api): def __init__(self, store, api):
self.store = store self.store = store
self.api = api self.api = api
self.stop_auto = False self.work = []
self.max_work = 100
self.unch_interval = 86400
self.atlas = self.store.get(Atlas, 'ATLAS', create=True)
def wait_for_stop(self): def find_work(self):
try: first_page = self.atlas.total_pages == 0
input() pages_left = self.atlas.total_pages < self.atlas.seen_pages
except EOFError: if first_page or pages_left:
pass self.sched(self.get_systems)
self.stop_auto = True return
print('stopping...') for s in self.store.all(System):
if len(self.work) > self.max_work:
def run(self, page=1):
print('universe mode. hit enter to stop')
t = Thread(target=self.wait_for_stop)
t.daemon = True
t.start()
self.all_systems(int(page))
print('manual mode')
def all_specials(self, waypoints):
for w in waypoints:
if self.stop_auto:
break break
if not s.uncharted: continue
if s.last_crawl > time() - self.unch_interval:
continue
self.sched(self.get_waypoints, s)
def do_work(self):
if len(self.work) == 0:
self.find_work()
if len(self.work) == 0:
return
work = self.work.pop()
work()
def get_systems(self):
page = 1
if self.atlas.seen_pages > 0:
page = self.atlas.seen_pages + 1
if page > self.atlas.total_pages:
return
data = self.api.list_systems(page)
self.atlas.total_pages = total_pages(self.api.last_meta)
self.atlas.seen_pages = page
def get_waypoints(self, system):
wps = self.api.list_waypoints(system)
system.last_crawl = time()
system.uncharted = len([1 for w in wps if w.uncharted]) > 0
self.schedule_specials(wps)
def sched(self, fun, *args):
self.work.append(partial(fun, *args))
def schedule_specials(self, waypoints):
for w in waypoints:
if 'UNCHARTED' in w.traits: if 'UNCHARTED' in w.traits:
continue continue
if 'MARKETPLACE' in w.traits: if 'MARKETPLACE' in w.traits:
print(f'marketplace at {w}') print(f'marketplace at {w}')
self.api.marketplace(w) self.sched(self.api.marketplace, w)
sleep(0.5)
if w.type == 'JUMP_GATE': if w.type == 'JUMP_GATE':
print(f'jumpgate at {w}') print(f'jumpgate at {w}')
self.api.jumps(w) self.sched(self.api.jumps, w)
if 'SHIPYARD' in w.traits:
def all_waypoints(self, systems): # todo
for s in systems: pass
if self.stop_auto:
break
r = self.api.list_waypoints(s)
self.all_specials(r)
sleep(0.5)
def all_systems(self, start_page):
self.stop_auto = False
data = self.api.list_systems(start_page)
pages = total_pages(self.api.last_meta)
print(f'{pages} pages of systems')
print(f'page {1}: {len(data)} results')
self.all_waypoints(data)
self.store.flush()
for p in range(start_page+1, pages+1):
if self.stop_auto:
break
data = self.api.list_systems(p)
print(f'page {p}: {len(data)} systems')
self.all_waypoints(data)
sleep(0.5)
self.store.flush()

View File

@ -4,6 +4,7 @@ from nullptr.missions import create_mission, get_mission_class
from random import choice from random import choice
from time import sleep from time import sleep
from threading import Thread from threading import Thread
from nullptr.atlas_builder import AtlasBuilder
class CentralCommandError(Exception): class CentralCommandError(Exception):
pass pass
@ -14,6 +15,7 @@ class CentralCommand:
self.stopping = False self.stopping = False
self.store = store self.store = store
self.api = api self.api = api
self.atlas_builder = AtlasBuilder(store, api)
self.update_missions() self.update_missions()
def get_ready_missions(self): def get_ready_missions(self):
@ -56,6 +58,9 @@ class CentralCommand:
request_counter = self.api.requests_sent request_counter = self.api.requests_sent
while request_counter == self.api.requests_sent and did_step: while request_counter == self.api.requests_sent and did_step:
did_step = self.tick() did_step = self.tick()
if request_counter == self.api.requests_sent:
self.atlas_builder.do_work()
self.store.flush() self.store.flush()
sleep(0.5) sleep(0.5)
self.stopping = False self.stopping = False

View File

@ -7,8 +7,8 @@ from nullptr.api import Api
from .util import * from .util import *
from time import sleep, time from time import sleep, time
from threading import Thread from threading import Thread
from nullptr.atlas_builder import AtlasBuilder
from nullptr.central_command import CentralCommand from nullptr.central_command import CentralCommand
class CommandError(Exception): class CommandError(Exception):
pass pass
@ -17,7 +17,6 @@ class Commander(CommandLine):
self.store = Store(store_file) self.store = Store(store_file)
self.agent = self.select_agent() self.agent = self.select_agent()
self.api = Api(self.store, self.agent) self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api)
self.centcom = CentralCommand(self.store, self.api) self.centcom = CentralCommand(self.store, self.api)
self.analyzer = Analyzer(self.store) self.analyzer = Analyzer(self.store)
self.ship = None self.ship = None
@ -183,9 +182,6 @@ class Commander(CommandLine):
self.api.register(faction.upper()) self.api.register(faction.upper())
pprint(self.api.agent) pprint(self.api.agent)
def do_universe(self, page=1):
self.atlas_builder.run(page)
def do_systems(self, page=1): def do_systems(self, page=1):
r = self.api.list_systems(int(page)) r = self.api.list_systems(int(page))
pprint(self.api.last_meta) pprint(self.api.last_meta)
@ -202,6 +198,11 @@ class Commander(CommandLine):
def do_defrag(self): def do_defrag(self):
self.store.defrag() self.store.defrag()
def do_system(self, system_str):
system = self.store.get(System, system_str)
r = self.api.list_waypoints(system)
pprint(r)
def do_waypoints(self, system_str=''): def do_waypoints(self, system_str=''):
if system_str == '': if system_str == '':
if not self.has_ship(): return if not self.has_ship(): return

View File

@ -8,5 +8,6 @@ from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship from nullptr.models.ship import Ship
from nullptr.models.contract import Contract from nullptr.models.contract import Contract
from nullptr.models.survey import Survey from nullptr.models.survey import Survey
from nullptr.models.atlas import Atlas
__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ] __all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base', 'Atlas' ]

11
nullptr/models/atlas.py Normal file
View File

@ -0,0 +1,11 @@
from .base import Base
class Atlas(Base):
@classmethod
def ext(self):
return 'atl'
def define(self):
self.total_pages = 0
self.seen_pages = 0

View File

@ -7,6 +7,8 @@ class System(Base):
self.x:int = 0 self.x:int = 0
self.y:int = 0 self.y:int = 0
self.type:str = 'unknown' self.type:str = 'unknown'
self.uncharted = True
self.last_crawl = 0
def update(self, d): def update(self, d):
self.seta('x', d) self.seta('x', d)

View File

@ -1,6 +1,7 @@
from .base import Base, Reference from .base import Base, Reference
from nullptr.models.system import System from nullptr.models.system import System
from nullptr.util import * from nullptr.util import *
from time import time
class Waypoint(Base): class Waypoint(Base):
def define(self): def define(self):
@ -10,6 +11,7 @@ class Waypoint(Base):
self.traits:list = [] self.traits:list = []
self.faction:str = '' self.faction:str = ''
self.system = self.get_system() self.system = self.get_system()
self.uncharted = True
def update(self, d): def update(self, d):
self.seta('x', d) self.seta('x', d)
@ -17,6 +19,7 @@ class Waypoint(Base):
self.seta('type', d) self.seta('type', d)
self.seta('faction', d, 'faction.symbol') self.seta('faction', d, 'faction.symbol')
self.setlst('traits', d, 'traits', 'symbol') self.setlst('traits', d, 'traits', 'symbol')
self.uncharted = 'UNCHARTED' in self.traits
@classmethod @classmethod
def ext(self): def ext(self):

View File

@ -113,7 +113,7 @@ class Store:
cnt += 1 cnt += 1
dur = time() - start_time dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds') print(f'Loaded {cnt} objects in {dur:.2f} seconds')
print(f'Fragmented space: {free} / {total} bytes') print(f'Fragmented space: {free} / {total} bytes')
def allocate_chunk(self, sz): def allocate_chunk(self, sz):