Update commander.py and store.py

nullptr/commander.py

@@ -16,10 +16,11 @@ class Commander(CommandLine):
   def __init__(self, store_dir='data', agent=None):
     self.store_dir = store_dir
     self.store = Store(store_dir)
+    self.store.load()
     self.agent = self.select_agent(agent)
     self.api = Api(self.store, self.agent)
     self.atlas_builder = AtlasBuilder(self.store, self.api)
-    self.store.flush()
+    
     self.stop_auto= False
     super().__init__()
   
@@ -27,7 +28,7 @@ class Commander(CommandLine):
     if agent_str is not None:
       return self.store.get(Agent, agent_str)
     else:
-      agents = self.store.all('', Agent)
+      agents = self.store.all(Agent)
       agent = next(agents, None)
       if agent is None:
         symbol = input('agent name: ')
@@ -67,25 +68,25 @@ class Commander(CommandLine):
   def do_test(self):
     start_time = time()
     total = 0
-    for m in self.store.all('', Marketplace, True):
+    for m in self.store.all(Marketplace):
       total += 1
     dur = time() - start_time
     print(f'{total} markets in {dur:.2f} seconds')
     start_time = time()
     total = 0
-    for m in self.store.all('', Jumpgate, True):
+    for m in self.store.all(Jumpgate):
       total += 1
     dur = time() - start_time
     print(f'{total} jumpgates in {dur:.2f} seconds')
     start_time = time()
     total = 0
-    for m in self.store.all('', Waypoint, True):
+    for m in self.store.all(Waypoint):
       total += 1
     dur = time() - start_time
     print(f'{total} waypoints in {dur:.2f} seconds')
     start_time = time()
     total = 0
-    for m in self.store.all('', System, True):
+    for m in self.store.all(System):
       total += 1
     dur = time() - start_time
     print(f'{total} systems in {dur:.2f} seconds')

nullptr/store.py

@@ -7,6 +7,7 @@ from nullptr.models.marketplace import Marketplace
 from nullptr.models.jumpgate import Jumpgate
 from os.path import isfile, dirname, isdir
 import os
+from os.path import basename
 import json
 from .util import *
 from time import time
@@ -15,44 +16,46 @@ class Store:
   def __init__(self, data_dir):
     self.init_models()
     self.data_dir = data_dir
-    self.data = {}
+    self.data = {m: {} for m in self.models}
     self.dirty_objects = set()
   
   def init_models(self):
     self.models = Base.__subclasses__()
     self.extensions = {c.ext(): c for c in self.models}
     
-  
   def dirty(self, obj):
     self.dirty_objects.add(obj)
     
   def path(self, obj):
     return os.path.join(self.data_dir, obj.path())
     
-  def load(self, obj):
+  def load_file(self, path):
-    path = self.path(obj)
     if not isfile(path):
-      return obj
+      return None
-    with open(path) as f:
+    fn = basename(path)
-      data = json.load(f)
+    ext = fn.split('.')[-1]
-    data['store'] = self
+    symbol = fn.split('.')[0]
-    obj.__dict__ = data
+    if ext not in self.extensions:
-  
-  def get_file(self, typ, path):
-    if not isfile(path):
-      print(path)
       return None
     with open(path) as f:
       data = json.load(f)
-    symbol = mg(data, 'symbol')
+    
-    oid = f'{symbol}.{typ.ext()}'
+    typ = self.extensions[ext]
-    if oid in self.data:
-      return self.data[oid]
     obj = typ(symbol, self)
-    self.load(obj)
+    data['store'] = self
-    self.data[oid] = obj
+    obj.__dict__ = data
+    self.data[typ][obj.symbol] = obj
     return obj
   
+  def load(self):
+    cnt = 0
+    start_time = time()
+    for fil in list_files(self.data_dir, True):
+      self.load_file(fil)
+      cnt += 1
+    dur = time() - start_time
+    print(f'loaded {cnt} objects in {dur:.2f} seconds')
+  
   def store(self, obj):
     path = self.path(obj)
     path_dir = dirname(path)
@@ -61,36 +64,33 @@ class Store:
       os.makedirs(path_dir, exist_ok=True)
     with open(path, 'w') as f:
       json.dump(data, f, indent=2)
-      
+  
-  def get(self, typ, symbol):
+  def create(self, typ, symbol):
-    oid = f'{symbol}.{typ.ext()}'
-    if oid in self.data:
-      return self.data[oid]
     obj = typ(symbol, self)
-    self.load(obj)
+    self.data[typ][symbol] = obj
-    self.data[oid] = obj
     return obj
+    
+  def get(self, typ, symbol, create=False):
+    if typ not in self.data:
+      return None
+    if symbol not in self.data[typ]:
+      if create:
+        return self.create(typ, symbol)
+      else:
+        return None
+    return self.data[typ][symbol]
   
   def update(self, typ, symbol, data):
-    obj = self.get(typ, symbol)
+    obj = self.get(typ, symbol, True)
     obj.update(data)
     return obj
   
   def update_list(self, typ, lst):
     return [self.update(typ, mg(d, 'symbol'), d) for d in lst]
     
-  def all(self, path, typ, recursive=False):
+  def all(self, typ):
-    if hasattr(path, 'path'):
+    for m in self.data[typ].values():
-      path = path.path()
+      yield m
-    path = os.path.join(self.data_dir, path)
-    if not isdir(path):
-      return
-    ext = '.' + typ.ext()
-    for fil in list_files(path, recursive):
-      if not fil.endswith(ext):
-        continue
-      yield self.get_file(typ, fil)
-    
   
   def flush(self):
     it = 0