from time import sleep, time from nullptr.util import * from threading import Thread from nullptr.models.atlas import Atlas from functools import partial from nullptr.models import System class AtlasBuilder: def __init__(self, store, api): self.store = store self.api = api self.work = [] self.max_work = 100 self.unch_interval = 86400 self.atlas = self.store.get(Atlas, 'ATLAS', create=True) def find_work(self): first_page = self.atlas.total_pages == 0 pages_left = self.atlas.total_pages > self.atlas.seen_pages if first_page or pages_left: self.sched(self.get_systems) return for s in self.store.all(System): if len(self.work) > self.max_work: break if not s.uncharted: continue if s.last_crawl > time() - self.unch_interval: continue self.sched(self.get_waypoints, s) def do_work(self): if len(self.work) == 0: self.find_work() if len(self.work) == 0: return work = self.work.pop() work() def get_systems(self): page = 1 if self.atlas.seen_pages > 0: page = self.atlas.seen_pages + 1 if page > self.atlas.total_pages: return # print('systems', page) data = self.api.list_systems(page) self.atlas.total_pages = total_pages(self.api.last_meta) self.atlas.seen_pages = page def get_waypoints(self, system): wps = self.api.list_waypoints(system) system.last_crawl = time() system.uncharted = len([1 for w in wps if w.uncharted]) > 0 self.schedule_specials(wps) def sched(self, fun, *args): self.work.append(partial(fun, *args)) def schedule_specials(self, waypoints): for w in waypoints: if 'UNCHARTED' in w.traits: continue if 'MARKETPLACE' in w.traits: # print(f'marketplace at {w}') self.sched(self.api.marketplace, w) if w.type == 'JUMP_GATE': # print(f'jumpgate at {w}') self.sched(self.api.jumps, w) if 'SHIPYARD' in w.traits: # todo pass