Rewrite atlas builder to be re-entrant. Rolled into automode.

nullptr/api.py

@@ -19,7 +19,7 @@ class Api:
     self.agent = agent
     self.store = store
     self.requests_sent = 0
-    self.meta = None
+    self.last_meta = None
     self.last_result = None
     self.root = 'https://api.spacetraders.io/v2/'
     

nullptr/atlas_builder.py

@@ -1,66 +1,71 @@
-from time import  sleep
+from time import  sleep, time
 from nullptr.util import *
 from threading import Thread
+from nullptr.models.atlas import Atlas
+from functools import partial
+from nullptr.models import System
 
 class AtlasBuilder:
   def __init__(self, store, api):
     self.store = store
     self.api = api
-    self.stop_auto = False
+    self.work = []
+    self.max_work = 100
+    self.unch_interval = 86400
+    self.atlas = self.store.get(Atlas, 'ATLAS', create=True)   
   
-  def wait_for_stop(self):
+  def find_work(self):
-    try:
+    first_page = self.atlas.total_pages == 0 
-      input()
+    pages_left = self.atlas.total_pages < self.atlas.seen_pages
-    except EOFError:
+    if first_page or pages_left:
-      pass
+      self.sched(self.get_systems)
-    self.stop_auto = True
+      return
-    print('stopping...')
+    for s in self.store.all(System):
-    
+      if len(self.work) > self.max_work:
-  def run(self, page=1):
-    print('universe mode. hit enter to stop')
-    t = Thread(target=self.wait_for_stop)
-    t.daemon = True
-    t.start()
-    self.all_systems(int(page))
-    print('manual mode')
-  
-  def all_specials(self, waypoints):
-    for w in waypoints:
-      if self.stop_auto:
         break
+      if not s.uncharted: continue
+      if s.last_crawl > time() - self.unch_interval: 
+        continue
+      self.sched(self.get_waypoints, s)
+      
+  
+  def do_work(self):
+    if len(self.work) == 0:
+      self.find_work()
+    if len(self.work) == 0:
+      return
+    work = self.work.pop()
+    work()
+  
+  def get_systems(self):
+    page = 1
+    if self.atlas.seen_pages > 0:
+      page = self.atlas.seen_pages + 1
+      if page > self.atlas.total_pages:
+        return
+    data = self.api.list_systems(page)
+    self.atlas.total_pages = total_pages(self.api.last_meta)
+    self.atlas.seen_pages = page
+  
+  def get_waypoints(self, system):
+    wps = self.api.list_waypoints(system)
+    system.last_crawl = time()
+    system.uncharted = len([1 for w in wps if w.uncharted]) > 0
+    self.schedule_specials(wps)
+ 
+  def sched(self, fun, *args):
+    self.work.append(partial(fun, *args))
+    
+  def schedule_specials(self, waypoints):
+    for w in waypoints:
       if 'UNCHARTED' in w.traits:
         continue
       if 'MARKETPLACE' in w.traits:
         print(f'marketplace at {w}')
-        self.api.marketplace(w)
+        self.sched(self.api.marketplace, w)
-        sleep(0.5)
       if w.type == 'JUMP_GATE':
         print(f'jumpgate at {w}')
-        self.api.jumps(w)
+        self.sched(self.api.jumps, w)
-        
+      if 'SHIPYARD' in w.traits:
-  def all_waypoints(self, systems):
+        # todo
-    for s in systems:
+        pass
-      if self.stop_auto:
-        break
-      r = self.api.list_waypoints(s)
-      self.all_specials(r)
-      sleep(0.5)
-      
-  
-  def all_systems(self, start_page):
-    self.stop_auto = False 
-    data = self.api.list_systems(start_page)
-    pages = total_pages(self.api.last_meta)
-    print(f'{pages} pages of systems')
-    print(f'page {1}: {len(data)} results')
-    self.all_waypoints(data)
-    self.store.flush()
-    
-    for p in range(start_page+1, pages+1):
-      if self.stop_auto:
-        break
-      data = self.api.list_systems(p)
-      print(f'page {p}: {len(data)} systems')
-      self.all_waypoints(data)
-      sleep(0.5)
-      self.store.flush()

nullptr/central_command.py

@@ -4,6 +4,7 @@ from nullptr.missions import create_mission, get_mission_class
 from random import choice
 from time import sleep
 from threading import Thread
+from nullptr.atlas_builder import  AtlasBuilder
 
 class CentralCommandError(Exception):
   pass
@@ -14,6 +15,7 @@ class CentralCommand:
     self.stopping = False
     self.store = store
     self.api = api
+    self.atlas_builder = AtlasBuilder(store, api)
     self.update_missions()
   
   def get_ready_missions(self):
@@ -56,6 +58,9 @@ class CentralCommand:
       request_counter = self.api.requests_sent
       while request_counter == self.api.requests_sent and did_step:
         did_step = self.tick()
+      if request_counter == self.api.requests_sent:
+        self.atlas_builder.do_work()
+        
       self.store.flush()
       sleep(0.5)
     self.stopping = False

nullptr/commander.py

@@ -7,8 +7,8 @@ from nullptr.api import Api
 from .util import *
 from time import sleep, time
 from threading import Thread
-from nullptr.atlas_builder import  AtlasBuilder
 from nullptr.central_command import CentralCommand
+
 class CommandError(Exception):
   pass
   
@@ -17,7 +17,6 @@ class Commander(CommandLine):
     self.store = Store(store_file)
     self.agent = self.select_agent()
     self.api = Api(self.store, self.agent)
-    self.atlas_builder = AtlasBuilder(self.store, self.api)
     self.centcom = CentralCommand(self.store, self.api)
     self.analyzer = Analyzer(self.store)
     self.ship = None
@@ -183,9 +182,6 @@ class Commander(CommandLine):
     self.api.register(faction.upper())
     pprint(self.api.agent)
     
-  def do_universe(self, page=1):
-    self.atlas_builder.run(page)
-  
   def do_systems(self, page=1):
     r = self.api.list_systems(int(page))
     pprint(self.api.last_meta)
@@ -202,6 +198,11 @@ class Commander(CommandLine):
   def do_defrag(self):
     self.store.defrag()
   
+  def do_system(self, system_str):
+    system = self.store.get(System, system_str)
+    r = self.api.list_waypoints(system)
+    pprint(r)
+    
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return

nullptr/models/__init__.py

@@ -8,5 +8,6 @@ from nullptr.models.jumpgate import Jumpgate
 from nullptr.models.ship import Ship
 from nullptr.models.contract import Contract
 from nullptr.models.survey import Survey
+from nullptr.models.atlas import Atlas
 
-__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]
+__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base', 'Atlas' ]

nullptr/models/atlas.py

@@ -0,0 +1,11 @@
+from .base import Base
+
+class Atlas(Base):
+  @classmethod
+  def ext(self):
+    return 'atl'
+  
+  def define(self):
+    self.total_pages = 0
+    self.seen_pages = 0
+    

nullptr/models/system.py

@@ -7,6 +7,8 @@ class System(Base):
     self.x:int = 0
     self.y:int = 0
     self.type:str = 'unknown'
+    self.uncharted = True
+    self.last_crawl = 0
   
   def update(self, d):
     self.seta('x', d)

nullptr/models/waypoint.py

@@ -1,6 +1,7 @@
 from .base import Base, Reference
 from nullptr.models.system import System
 from nullptr.util import *
+from time import time
 
 class Waypoint(Base):
   def define(self):
@@ -10,6 +11,7 @@ class Waypoint(Base):
     self.traits:list = []
     self.faction:str = ''
     self.system = self.get_system()
+    self.uncharted = True
     
   def update(self, d):
     self.seta('x', d)
@@ -17,6 +19,7 @@ class Waypoint(Base):
     self.seta('type', d)
     self.seta('faction', d, 'faction.symbol')
     self.setlst('traits', d, 'traits', 'symbol')
+    self.uncharted = 'UNCHARTED' in self.traits
     
   @classmethod
   def ext(self):

nullptr/store.py

@@ -113,7 +113,7 @@ class Store:
       cnt += 1
       
     dur = time() - start_time
-    print(f'loaded {cnt} objects in {dur:.2f} seconds')
+    print(f'Loaded {cnt} objects in {dur:.2f} seconds')
     print(f'Fragmented space: {free} / {total} bytes')
   
   def allocate_chunk(self, sz):