Update commander.py and store.py

This commit is contained in:
Richard Bronkhorst 2023-06-12 23:13:40 +02:00
parent 6434ba8a6a
commit c7739c5985
2 changed files with 46 additions and 45 deletions

View File

@ -16,10 +16,11 @@ class Commander(CommandLine):
def __init__(self, store_dir='data', agent=None): def __init__(self, store_dir='data', agent=None):
self.store_dir = store_dir self.store_dir = store_dir
self.store = Store(store_dir) self.store = Store(store_dir)
self.store.load()
self.agent = self.select_agent(agent) self.agent = self.select_agent(agent)
self.api = Api(self.store, self.agent) self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api) self.atlas_builder = AtlasBuilder(self.store, self.api)
self.store.flush()
self.stop_auto= False self.stop_auto= False
super().__init__() super().__init__()
@ -27,7 +28,7 @@ class Commander(CommandLine):
if agent_str is not None: if agent_str is not None:
return self.store.get(Agent, agent_str) return self.store.get(Agent, agent_str)
else: else:
agents = self.store.all('', Agent) agents = self.store.all(Agent)
agent = next(agents, None) agent = next(agents, None)
if agent is None: if agent is None:
symbol = input('agent name: ') symbol = input('agent name: ')
@ -67,25 +68,25 @@ class Commander(CommandLine):
def do_test(self): def do_test(self):
start_time = time() start_time = time()
total = 0 total = 0
for m in self.store.all('', Marketplace, True): for m in self.store.all(Marketplace):
total += 1 total += 1
dur = time() - start_time dur = time() - start_time
print(f'{total} markets in {dur:.2f} seconds') print(f'{total} markets in {dur:.2f} seconds')
start_time = time() start_time = time()
total = 0 total = 0
for m in self.store.all('', Jumpgate, True): for m in self.store.all(Jumpgate):
total += 1 total += 1
dur = time() - start_time dur = time() - start_time
print(f'{total} jumpgates in {dur:.2f} seconds') print(f'{total} jumpgates in {dur:.2f} seconds')
start_time = time() start_time = time()
total = 0 total = 0
for m in self.store.all('', Waypoint, True): for m in self.store.all(Waypoint):
total += 1 total += 1
dur = time() - start_time dur = time() - start_time
print(f'{total} waypoints in {dur:.2f} seconds') print(f'{total} waypoints in {dur:.2f} seconds')
start_time = time() start_time = time()
total = 0 total = 0
for m in self.store.all('', System, True): for m in self.store.all(System):
total += 1 total += 1
dur = time() - start_time dur = time() - start_time
print(f'{total} systems in {dur:.2f} seconds') print(f'{total} systems in {dur:.2f} seconds')

View File

@ -7,6 +7,7 @@ from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate from nullptr.models.jumpgate import Jumpgate
from os.path import isfile, dirname, isdir from os.path import isfile, dirname, isdir
import os import os
from os.path import basename
import json import json
from .util import * from .util import *
from time import time from time import time
@ -15,44 +16,46 @@ class Store:
def __init__(self, data_dir): def __init__(self, data_dir):
self.init_models() self.init_models()
self.data_dir = data_dir self.data_dir = data_dir
self.data = {} self.data = {m: {} for m in self.models}
self.dirty_objects = set() self.dirty_objects = set()
def init_models(self): def init_models(self):
self.models = Base.__subclasses__() self.models = Base.__subclasses__()
self.extensions = {c.ext(): c for c in self.models} self.extensions = {c.ext(): c for c in self.models}
def dirty(self, obj): def dirty(self, obj):
self.dirty_objects.add(obj) self.dirty_objects.add(obj)
def path(self, obj): def path(self, obj):
return os.path.join(self.data_dir, obj.path()) return os.path.join(self.data_dir, obj.path())
def load(self, obj): def load_file(self, path):
path = self.path(obj)
if not isfile(path): if not isfile(path):
return obj return None
with open(path) as f: fn = basename(path)
data = json.load(f) ext = fn.split('.')[-1]
data['store'] = self symbol = fn.split('.')[0]
obj.__dict__ = data if ext not in self.extensions:
def get_file(self, typ, path):
if not isfile(path):
print(path)
return None return None
with open(path) as f: with open(path) as f:
data = json.load(f) data = json.load(f)
symbol = mg(data, 'symbol')
oid = f'{symbol}.{typ.ext()}' typ = self.extensions[ext]
if oid in self.data:
return self.data[oid]
obj = typ(symbol, self) obj = typ(symbol, self)
self.load(obj) data['store'] = self
self.data[oid] = obj obj.__dict__ = data
self.data[typ][obj.symbol] = obj
return obj return obj
def load(self):
cnt = 0
start_time = time()
for fil in list_files(self.data_dir, True):
self.load_file(fil)
cnt += 1
dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds')
def store(self, obj): def store(self, obj):
path = self.path(obj) path = self.path(obj)
path_dir = dirname(path) path_dir = dirname(path)
@ -61,36 +64,33 @@ class Store:
os.makedirs(path_dir, exist_ok=True) os.makedirs(path_dir, exist_ok=True)
with open(path, 'w') as f: with open(path, 'w') as f:
json.dump(data, f, indent=2) json.dump(data, f, indent=2)
def get(self, typ, symbol): def create(self, typ, symbol):
oid = f'{symbol}.{typ.ext()}'
if oid in self.data:
return self.data[oid]
obj = typ(symbol, self) obj = typ(symbol, self)
self.load(obj) self.data[typ][symbol] = obj
self.data[oid] = obj
return obj return obj
def get(self, typ, symbol, create=False):
if typ not in self.data:
return None
if symbol not in self.data[typ]:
if create:
return self.create(typ, symbol)
else:
return None
return self.data[typ][symbol]
def update(self, typ, symbol, data): def update(self, typ, symbol, data):
obj = self.get(typ, symbol) obj = self.get(typ, symbol, True)
obj.update(data) obj.update(data)
return obj return obj
def update_list(self, typ, lst): def update_list(self, typ, lst):
return [self.update(typ, mg(d, 'symbol'), d) for d in lst] return [self.update(typ, mg(d, 'symbol'), d) for d in lst]
def all(self, path, typ, recursive=False): def all(self, typ):
if hasattr(path, 'path'): for m in self.data[typ].values():
path = path.path() yield m
path = os.path.join(self.data_dir, path)
if not isdir(path):
return
ext = '.' + typ.ext()
for fil in list_files(path, recursive):
if not fil.endswith(ext):
continue
yield self.get_file(typ, fil)
def flush(self): def flush(self):
it = 0 it = 0