Update atlas_builder.py

nullptr/api.py

@@ -19,7 +19,7 @@ class Api:
     self.agent = agent
     self.store = store
     self.requests_sent = 0
-    self.meta = None
+    self.last_meta = None
     self.last_result = None
     self.root = 'https://api.spacetraders.io/v2/'
     
@@ -235,7 +235,7 @@ class Api:
     return ship
     
   def shipyard(self, wp):
-    return self.request('get', f'systems/{wp.system()}/waypoints/{wp}/shipyard')
+    return self.request('get', f'systems/{wp.system}/waypoints/{wp}/shipyard')
 
   def extract(self, ship, survey=None):
     data = {}

nullptr/atlas_builder.py

@@ -1,66 +1,71 @@
-from time import  sleep
+from time import  sleep, time
 from nullptr.util import *
 from threading import Thread
+from nullptr.models.atlas import Atlas
+from functools import partial
+from nullptr.models import System
 
 class AtlasBuilder:
   def __init__(self, store, api):
     self.store = store
     self.api = api
-    self.stop_auto = False
+    self.work = []
+    self.max_work = 100
+    self.unch_interval = 86400
+    self.atlas = self.store.get(Atlas, 'ATLAS', create=True)   
   
-  def wait_for_stop(self):
+  def find_work(self):
-    try:
+    first_page = self.atlas.total_pages == 0 
-      input()
+    pages_left = self.atlas.total_pages < self.atlas.seen_pages
-    except EOFError:
+    if first_page or pages_left:
-      pass
+      self.sched(self.get_systems)
-    self.stop_auto = True
+      return
-    print('stopping...')
+    for s in self.store.all(System):
-    
+      if len(self.work) > self.max_work:
-  def run(self, page=1):
-    print('universe mode. hit enter to stop')
-    t = Thread(target=self.wait_for_stop)
-    t.daemon = True
-    t.start()
-    self.all_systems(int(page))
-    print('manual mode')
-  
-  def all_specials(self, waypoints):
-    for w in waypoints:
-      if self.stop_auto:
         break
+      if not s.uncharted: continue
+      if s.last_crawl > time() - self.unch_interval: 
+        continue
+      self.sched(self.get_waypoints, s)
+      
+  
+  def do_work(self):
+    if len(self.work) == 0:
+      self.find_work()
+    if len(self.work) == 0:
+      return
+    work = self.work.pop()
+    work()
+  
+  def get_systems(self):
+    page = 1
+    if self.atlas.seen_pages > 0:
+      page = self.atlas.seen_pages + 1
+      if page > self.atlas.total_pages:
+        return
+    data = self.api.list_systems(page)
+    self.atlas.total_pages = total_pages(self.api.last_meta)
+    self.atlas.seen_pages = page
+  
+  def get_waypoints(self, system):
+    wps = self.api.list_waypoints(system)
+    system.last_crawl = time()
+    system.uncharted = len([1 for w in wps if w.uncharted]) > 0
+    self.schedule_specials(wps)
+ 
+  def sched(self, fun, *args):
+    self.work.append(partial(fun, *args))
+    
+  def schedule_specials(self, waypoints):
+    for w in waypoints:
       if 'UNCHARTED' in w.traits:
         continue
       if 'MARKETPLACE' in w.traits:
-        print(f'marketplace at {w}')
+        #  print(f'marketplace at {w}')
-        self.api.marketplace(w)
+        self.sched(self.api.marketplace, w)
-        sleep(0.5)
       if w.type == 'JUMP_GATE':
-        print(f'jumpgate at {w}')
+        # print(f'jumpgate at {w}')
-        self.api.jumps(w)
+        self.sched(self.api.jumps, w)
-        
+      if 'SHIPYARD' in w.traits:
-  def all_waypoints(self, systems):
+        # todo
-    for s in systems:
+        pass
-      if self.stop_auto:
-        break
-      r = self.api.list_waypoints(s)
-      self.all_specials(r)
-      sleep(0.5)
-      
-  
-  def all_systems(self, start_page):
-    self.stop_auto = False 
-    data = self.api.list_systems(start_page)
-    pages = total_pages(self.api.last_meta)
-    print(f'{pages} pages of systems')
-    print(f'page {1}: {len(data)} results')
-    self.all_waypoints(data)
-    self.store.flush()
-    
-    for p in range(start_page+1, pages+1):
-      if self.stop_auto:
-        break
-      data = self.api.list_systems(p)
-      print(f'page {p}: {len(data)} systems')
-      self.all_waypoints(data)
-      sleep(0.5)
-      self.store.flush()

nullptr/central_command.py

@@ -4,6 +4,7 @@ from nullptr.missions import create_mission, get_mission_class
 from random import choice
 from time import sleep
 from threading import Thread
+from nullptr.atlas_builder import  AtlasBuilder
 
 class CentralCommandError(Exception):
   pass
@@ -14,6 +15,7 @@ class CentralCommand:
     self.stopping = False
     self.store = store
     self.api = api
+    self.atlas_builder = AtlasBuilder(store, api)
     self.update_missions()
   
   def get_ready_missions(self):
@@ -56,6 +58,9 @@ class CentralCommand:
       request_counter = self.api.requests_sent
       while request_counter == self.api.requests_sent and did_step:
         did_step = self.tick()
+      if request_counter == self.api.requests_sent:
+        self.atlas_builder.do_work()
+        
       self.store.flush()
       sleep(0.5)
     self.stopping = False

nullptr/commander.py

@@ -7,8 +7,8 @@ from nullptr.api import Api
 from .util import *
 from time import sleep, time
 from threading import Thread
-from nullptr.atlas_builder import  AtlasBuilder
 from nullptr.central_command import CentralCommand
+
 class CommandError(Exception):
   pass
   
@@ -17,7 +17,6 @@ class Commander(CommandLine):
     self.store = Store(store_file)
     self.agent = self.select_agent()
     self.api = Api(self.store, self.agent)
-    self.atlas_builder = AtlasBuilder(self.store, self.api)
     self.centcom = CentralCommand(self.store, self.api)
     self.analyzer = Analyzer(self.store)
     self.ship = None
@@ -56,6 +55,7 @@ class Commander(CommandLine):
   def agent_setup(self):
     symbol = input('agent name: ')
     agent = self.store.get(Agent, symbol, create=True)
+    self.agent = agent
     api = Api(self.store, agent)
     self.api = api
     faction = input('faction: ')
@@ -66,6 +66,8 @@ class Commander(CommandLine):
     self.do_ships('r')
     print('=== contracts')
     self.do_contracts('r')
+    ship = self.store.get(Ship, symbol.upper() + '-2')
+    api.list_waypoints(ship.location.system)
     self.store.flush()
     return agent
   
@@ -183,9 +185,6 @@ class Commander(CommandLine):
     self.api.register(faction.upper())
     pprint(self.api.agent)
     
-  def do_universe(self, page=1):
-    self.atlas_builder.run(page)
-  
   def do_systems(self, page=1):
     r = self.api.list_systems(int(page))
     pprint(self.api.last_meta)
@@ -199,6 +198,14 @@ class Commander(CommandLine):
       print(f'{num:5d} {nam}')
     print(f'{total:5d} total')
   
+  def do_defrag(self):
+    self.store.defrag()
+  
+  def do_system(self, system_str):
+    system = self.store.get(System, system_str)
+    r = self.api.list_waypoints(system)
+    pprint(r)
+    
   def do_waypoints(self, system_str=''):
     if system_str == '':
       if not self.has_ship(): return

nullptr/missions/base.py

@@ -163,7 +163,7 @@ class BaseMission(Mission):
 
   def step_sell(self, except_resource=True):
     target = self.st('resource')
-    market = self.store.get('Marketplace', self.ship.location_str)
+    market = self.store.get('Marketplace', self.ship.location.symbol)
     sellables = market.sellable_items(self.ship.cargo.keys())
     if target in sellables and except_resource:
       sellables.remove(target)

nullptr/models/__init__.py

@@ -8,5 +8,6 @@ from nullptr.models.jumpgate import Jumpgate
 from nullptr.models.ship import Ship
 from nullptr.models.contract import Contract
 from nullptr.models.survey import Survey
+from nullptr.models.atlas import Atlas
 
-__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]
+__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'System', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base', 'Atlas' ]

nullptr/models/atlas.py

@@ -0,0 +1,11 @@
+from .base import Base
+
+class Atlas(Base):
+  @classmethod
+  def ext(self):
+    return 'atl'
+  
+  def define(self):
+    self.total_pages = 0
+    self.seen_pages = 0
+    

nullptr/models/system.py

@@ -7,6 +7,8 @@ class System(Base):
     self.x:int = 0
     self.y:int = 0
     self.type:str = 'unknown'
+    self.uncharted = True
+    self.last_crawl = 0
   
   def update(self, d):
     self.seta('x', d)

nullptr/models/waypoint.py

@@ -1,6 +1,7 @@
 from .base import Base, Reference
 from nullptr.models.system import System
 from nullptr.util import *
+from time import time
 
 class Waypoint(Base):
   def define(self):
@@ -10,6 +11,7 @@ class Waypoint(Base):
     self.traits:list = []
     self.faction:str = ''
     self.system = self.get_system()
+    self.uncharted = True
     
   def update(self, d):
     self.seta('x', d)
@@ -17,6 +19,7 @@ class Waypoint(Base):
     self.seta('type', d)
     self.seta('faction', d, 'faction.symbol')
     self.setlst('traits', d, 'traits', 'symbol')
+    self.uncharted = 'UNCHARTED' in self.traits
     
   @classmethod
   def ext(self):

nullptr/store.py

@@ -95,13 +95,16 @@ class Store:
   def load(self):
     cnt = 0
     start_time = time()
-    
+    total = 0
+    free = 0
     self.fil.seek(0)
     offset = 0
     while (hdr := ChunkHeader.parse(self.fil)):
       # print(hdr)
+      total += hdr.size
       if not hdr.in_use: 
         self.fil.seek(hdr.size, 1)
+        free += hdr.size
         continue
       data = self.fil.read(hdr.used)
       self.load_object(data, offset)
@@ -110,7 +113,8 @@ class Store:
       cnt += 1
       
     dur = time() - start_time
-    print(f'loaded {cnt} objects in {dur:.2f} seconds')
+    print(f'Loaded {cnt} objects in {dur:.2f} seconds')
+    print(f'Fragmented space: {free} / {total} bytes')
   
   def allocate_chunk(self, sz):
     used = sz
@@ -127,6 +131,16 @@ class Store:
     h.write(self.fil)
     return offset, h
     
+  def purge(self, obj):
+    if obj.file_offset is None:
+      return
+    self.fil.seek(obj.file_offset)
+    hdr = ChunkHeader.parse(self.fil)
+    hdr.in_use = False
+    self.fil.seek(obj.file_offset)
+    hdr.write(self.fil)
+    obj.file_offset = None
+    
   def store(self, obj):
     data = self.dump_object(obj)
     osize = len(data)
@@ -207,6 +221,9 @@ class Store:
       typ = self.model_names[typ]
     
     for m in self.data[typ].values():
+      if m.is_expired():
+        self.dirty(m)
+        continue
       yield m
   
   def all_members(self, system, typ=None):
@@ -219,10 +236,18 @@ class Store:
     if system not in self.system_members:
       return 
     
+    garbage = set()
     for m in self.system_members[system]:
+      if m.is_expired():
+        self.dirty(m)
+        garbage.add(m)
+        continue
       if typ is None or type(m) == typ:
         yield m
     
+    for m in garbage:
+      self.system_members[system].remove(m)
+      
   def cleanup(self):
     if time() < self.last_cleanup + self.cleanup_interval:
       return
@@ -233,8 +258,7 @@ class Store:
         if o.is_expired():
           expired.append(o)
     for o in expired:
-      
+      self.purge(obj)
-      # TODO
       
       del self.data[type(o)][o.symbol]
     dur = time() - start_time
@@ -246,8 +270,21 @@ class Store:
     start_time = time()
     for obj in self.dirty_objects:
       it += 1
-      self.store(obj)
+      if obj.is_expired():
+        self.purge(obj)
+      else:
+        self.store(obj)
     self.fil.flush()
     self.dirty_objects = set()
     dur = time() - start_time
     #  print(f'flush done {it} items {dur:.2f}')
+  
+  def defrag(self):
+    nm = self.fil.name
+    self.fil.close()
+    os.rename(nm, nm + '.bak')
+    self.fil = open(nm, 'ab+')
+    for t in self.data:
+      for o in self.all(t):
+        o.file_offset = None
+        self.store(o)

store.md

@@ -34,8 +34,8 @@ An index is a dict with a string as key and a list of objects as value. The dict
 * store.load(fil) loads all objects
 * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
 * store.all(type) generator for all objects of a goven type
-* store.delete(typ, symbol)
+* store.purge(obj)
-* store.cleanup() removes all expired objects
+* store.clean() removes all expired objects
 * store.flush() writes all dirty objects to disk
 * store.defrag() consolidates the store file to minimize storage and loading time