Update atlas_builder.py, central_command.py and one other file

This commit is contained in:
Richard Bronkhorst 2023-07-18 12:43:31 +02:00
parent 71f8eb9ed8
commit 2716fbf1aa
3 changed files with 8 additions and 3 deletions

View File

@ -16,7 +16,8 @@ class AtlasBuilder:
def find_work(self):
first_page = self.atlas.total_pages == 0
pages_left = self.atlas.total_pages < self.atlas.seen_pages
pages_left = self.atlas.total_pages > self.atlas.seen_pages
if first_page or pages_left:
self.sched(self.get_systems)
return
@ -43,6 +44,7 @@ class AtlasBuilder:
page = self.atlas.seen_pages + 1
if page > self.atlas.total_pages:
return
# print('systems', page)
data = self.api.list_systems(page)
self.atlas.total_pages = total_pages(self.api.last_meta)
self.atlas.seen_pages = page

View File

@ -60,6 +60,8 @@ class CentralCommand:
did_step = self.tick()
if request_counter == self.api.requests_sent:
self.atlas_builder.do_work()
else:
print('nowork')
self.store.flush()
sleep(0.5)

View File

@ -251,6 +251,7 @@ class Store:
def cleanup(self):
if time() < self.last_cleanup + self.cleanup_interval:
return
self.last_cleanup = time()
start_time = time()
expired = list()
for t in self.data:
@ -262,7 +263,7 @@ class Store:
del self.data[type(o)][o.symbol]
dur = time() - start_time
# print(f'cleaned {len(expired)} in {dur:.03f} seconds')
print(f'cleaned {len(expired)} in {dur:.03f} seconds')
def flush(self):
self.cleanup()
@ -277,7 +278,7 @@ class Store:
self.fil.flush()
self.dirty_objects = set()
dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}')
# print(f'flush done {it} items {dur:.2f}')
def defrag(self):
nm = self.fil.name