New store setup

This commit is contained in:
Richard Bronkhorst 2023-07-10 19:25:01 +02:00
parent 6537db3c03
commit b1e3621490
14 changed files with 212 additions and 136 deletions

View File

@ -9,4 +9,4 @@ ADD --chown=user . /app
RUN chmod +x /app/main.py RUN chmod +x /app/main.py
VOLUME /data VOLUME /data
ENTRYPOINT [ "python3", "/app/main.py"] ENTRYPOINT [ "python3", "/app/main.py"]
CMD ["-s", "/data/"] CMD ["-s", "/data/store.npt"]

View File

@ -1,8 +1,8 @@
import argparse import argparse
from nullptr.commander import Commander from nullptr.commander import Commander
from nullptr.models.base import Base
def main(args): def main(args):
c = Commander(args.store_dir) c = Commander(args.store_file)
c.run() c.run()
# X1-AG74-41076A # X1-AG74-41076A
@ -10,6 +10,6 @@ def main(args):
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('-s', '--store-dir', default='data') parser.add_argument('-s', '--store-file', default='data/store.npt')
args = parser.parse_args() args = parser.parse_args()
main(args) main(args)

View File

@ -69,6 +69,7 @@ class Api:
} }
result = self.request('post', 'register', data, need_token=False) result = self.request('post', 'register', data, need_token=False)
token = mg(result, 'token') token = mg(result, 'token')
self.agent.update(mg(result, 'agent'))
self.agent.token = token self.agent.token = token
def info(self): def info(self):
@ -87,13 +88,12 @@ class Api:
return self.store.update_list(Waypoint, data) return self.store.update_list(Waypoint, data)
def marketplace(self, waypoint): def marketplace(self, waypoint):
system = waypoint.system() system = waypoint.system
symbol = str(waypoint)
data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market') data = self.request('get', f'systems/{system}/waypoints/{waypoint}/market')
return self.store.update(Marketplace, data) return self.store.update(Marketplace, data)
def jumps(self, waypoint): def jumps(self, waypoint):
data = self.request('get', f'systems/{waypoint.system()}/waypoints/{waypoint}/jump-gate') data = self.request('get', f'systems/{waypoint.system}/waypoints/{waypoint}/jump-gate')
symbol = str(waypoint) symbol = str(waypoint)
return self.store.update(Jumpgate, data, symbol) return self.store.update(Jumpgate, data, symbol)

View File

@ -2,11 +2,7 @@ from nullptr.command_line import CommandLine
from nullptr.store import Store from nullptr.store import Store
from nullptr.analyzer import Analyzer from nullptr.analyzer import Analyzer
import argparse import argparse
from nullptr.models.agent import Agent from nullptr.models import *
from nullptr.models.system import System
from nullptr.models.waypoint import Waypoint
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.api import Api from nullptr.api import Api
from .util import * from .util import *
from time import sleep, time from time import sleep, time
@ -17,10 +13,8 @@ class CommandError(Exception):
pass pass
class Commander(CommandLine): class Commander(CommandLine):
def __init__(self, store_dir='data'): def __init__(self, store_file='data/store.npt'):
self.store_dir = store_dir self.store = Store(store_file)
self.store = Store(store_dir)
self.store.load()
self.agent = self.select_agent() self.agent = self.select_agent()
self.api = Api(self.store, self.agent) self.api = Api(self.store, self.agent)
self.atlas_builder = AtlasBuilder(self.store, self.api) self.atlas_builder = AtlasBuilder(self.store, self.api)
@ -58,6 +52,10 @@ class Commander(CommandLine):
if agent is None: if agent is None:
symbol = input('agent name: ') symbol = input('agent name: ')
agent = self.store.get(Agent, symbol, create=True) agent = self.store.get(Agent, symbol, create=True)
api = Api(self.store, agent)
faction = input('faction: ')
api.register(faction.upper().strip())
self.store.flush()
return agent return agent
def resolve(self, typ, arg): def resolve(self, typ, arg):
@ -107,7 +105,7 @@ class Commander(CommandLine):
def do_cmine(self): def do_cmine(self):
if not self.has_ship(): return if not self.has_ship(): return
site = self.ship.location_str site = self.ship.location
contract = self.active_contract() contract = self.active_contract()
delivery = contract.unfinished_delivery() delivery = contract.unfinished_delivery()
if delivery is None: if delivery is None:
@ -118,7 +116,7 @@ class Commander(CommandLine):
self.centcom.set_mission_param(self.ship, 'site', site) self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource) self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination) self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol) self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission() self.print_mission()
def do_chaul(self): def do_chaul(self):
@ -140,10 +138,10 @@ class Commander(CommandLine):
_, m, _, _ = m[0] _, m, _, _ = m[0]
site = self.store.get(Waypoint, m.symbol) site = self.store.get(Waypoint, m.symbol)
self.centcom.init_mission(self.ship, 'haul') self.centcom.init_mission(self.ship, 'haul')
self.centcom.set_mission_param(self.ship, 'site', site.symbol) self.centcom.set_mission_param(self.ship, 'site', site)
self.centcom.set_mission_param(self.ship, 'resource', resource) self.centcom.set_mission_param(self.ship, 'resource', resource)
self.centcom.set_mission_param(self.ship, 'dest', destination) self.centcom.set_mission_param(self.ship, 'dest', destination)
self.centcom.set_mission_param(self.ship, 'contract', contract.symbol) self.centcom.set_mission_param(self.ship, 'contract', contract)
self.print_mission() self.print_mission()
def do_cprobe(self): def do_cprobe(self):
@ -160,15 +158,14 @@ class Commander(CommandLine):
return return
markets = [ mkt[1] for mkt in m] markets = [ mkt[1] for mkt in m]
markets = self.analyzer.solve_tsp(markets) markets = self.analyzer.solve_tsp(markets)
hops = ','.join([m.symbol for m in markets])
self.centcom.init_mission(self.ship, 'probe') self.centcom.init_mission(self.ship, 'probe')
self.centcom.set_mission_param(self.ship, 'hops', hops) self.centcom.set_mission_param(self.ship, 'hops', markets)
self.print_mission() self.print_mission()
def do_travel(self, dest): def do_travel(self, dest):
dest = self.resolve('Waypoint', dest) dest = self.resolve('Waypoint', dest)
self.centcom.init_mission(self.ship, 'travel') self.centcom.init_mission(self.ship, 'travel')
self.centcom.set_mission_param(self.ship, 'dest', dest.symbol) self.centcom.set_mission_param(self.ship, 'dest', dest)
self.print_mission() self.print_mission()
def do_register(self, faction): def do_register(self, faction):
@ -194,7 +191,7 @@ class Commander(CommandLine):
def do_waypoints(self, system_str=''): def do_waypoints(self, system_str=''):
if system_str == '': if system_str == '':
if not self.has_ship(): return if not self.has_ship(): return
system = self.ship.location().system() system = self.ship.location.system
else: else:
system = self.store.get(System, system_str) system = self.store.get(System, system_str)
r = self.store.all_members(system, 'Waypoint') r = self.store.all_members(system, 'Waypoint')
@ -220,7 +217,7 @@ class Commander(CommandLine):
def do_jumps(self, waypoint_str=None): def do_jumps(self, waypoint_str=None):
if waypoint_str is None: if waypoint_str is None:
if not self.has_ship(): return if not self.has_ship(): return
waypoint = self.ship.location() waypoint = self.ship.location
else: else:
waypoint = self.store.get(Waypoint, waypoint_str.upper()) waypoint = self.store.get(Waypoint, waypoint_str.upper())
r = self.api.jumps(waypoint) r = self.api.jumps(waypoint)
@ -228,7 +225,7 @@ class Commander(CommandLine):
def do_query(self, resource): def do_query(self, resource):
if not self.has_ship(): return if not self.has_ship(): return
location = self.ship.location() location = self.ship.location
resource = resource.upper() resource = resource.upper()
print('Found markets:') print('Found markets:')
for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location): for typ, m, d, plen in self.analyzer.find_closest_markets(resource, 'buy,exchange',location):
@ -261,7 +258,7 @@ class Commander(CommandLine):
def do_deliver(self): def do_deliver(self):
if not self.has_ship(): return if not self.has_ship(): return
site = self.ship.location_str site = self.ship.location
contract = self.active_contract() contract = self.active_contract()
delivery = contract.unfinished_delivery() delivery = contract.unfinished_delivery()
if delivery is None: if delivery is None:
@ -290,7 +287,7 @@ class Commander(CommandLine):
def do_go(self, arg): def do_go(self, arg):
if not self.has_ship(): return if not self.has_ship(): return
system = self.ship.location().system() system = self.ship.location.system
symbol = f'{system}-{arg}' symbol = f'{system}-{arg}'
dest = self.resolve('Waypoint', symbol) dest = self.resolve('Waypoint', symbol)
self.api.navigate(self.ship, dest) self.api.navigate(self.ship, dest)
@ -324,7 +321,7 @@ class Commander(CommandLine):
def do_market(self, arg=''): def do_market(self, arg=''):
if arg == '': if arg == '':
if not self.has_ship(): return if not self.has_ship(): return
waypoint = self.ship.location() waypoint = self.ship.location
else: else:
waypoint = self.resolve('Waypoint', arg) waypoint = self.resolve('Waypoint', arg)
r = self.api.marketplace(waypoint) r = self.api.marketplace(waypoint)
@ -354,7 +351,7 @@ class Commander(CommandLine):
def do_shipyard(self): def do_shipyard(self):
if not self.has_ship(): return if not self.has_ship(): return
location = self.ship.location() location = self.ship.location
data = self.api.shipyard(location) data = self.api.shipyard(location)
for s in must_get(data, 'ships'): for s in must_get(data, 'ships'):
print(s['type'], s['purchasePrice']) print(s['type'], s['purchasePrice'])
@ -362,7 +359,7 @@ class Commander(CommandLine):
def do_jump(self, system_str): def do_jump(self, system_str):
if not self.has_ship(): return if not self.has_ship(): return
if '-' not in system_str: if '-' not in system_str:
sector = self.ship.location_str.split('-')[0] sector = self.ship.location.system.sector.symbol
system_str = f'{sector}-{system_str}' system_str = f'{sector}-{system_str}'
system = self.resolve('System', system_str) system = self.resolve('System', system_str)
self.api.jump(self.ship, system) self.api.jump(self.ship, system)
@ -370,7 +367,7 @@ class Commander(CommandLine):
def do_purchase(self, ship_type): def do_purchase(self, ship_type):
if not self.has_ship(): return if not self.has_ship(): return
location = self.ship.location() location = self.ship.location
ship_type = ship_type.upper() ship_type = ship_type.upper()
if not ship_type.startswith('SHIP'): if not ship_type.startswith('SHIP'):
ship_type = 'SHIP_' + ship_type ship_type = 'SHIP_' + ship_type

View File

@ -0,0 +1,12 @@
from nullptr.models.base import Base
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
__all__ = [ 'Waypoint', 'Sector', 'Ship', 'Survey', 'Agent', 'Marketplace', 'Jumpgate', 'Contract', 'Base' ]

View File

@ -1,18 +1,38 @@
from copy import deepcopy from copy import deepcopy
from nullptr.util import sg from nullptr.util import sg
class Reference:
def __init__(self, typ, symbol, store):
self.typ = typ
self.symbol = symbol
self.store = store
@classmethod
def create(cls, obj):
o = cls(type(obj), obj.symbol, obj.store)
return o
def resolve(self):
self.store.get(self.typ, self.symbol)
def __repr__(self):
return f'*REF*{self.symbol}.{self.typ.ext()}'
class Base: class Base:
identifier = 'symbol' identifier = 'symbol'
symbol: str
store: object
def __init__(self, symbol, store): def __init__(self, symbol, store):
self.disable_dirty = True self.disable_dirty = True
self.file_offset = 0
self.store = store self.store = store
self.symbol = symbol self.symbol = symbol
self.define() self.define()
self.disable_dirty = False self.disable_dirty = False
@classmethod
def ext(cls):
raise NotImplementedError('no ext')
def define(self): def define(self):
pass pass
@ -38,10 +58,18 @@ class Base:
setattr(self, attr, lst) setattr(self, attr, lst)
def __setattr__(self, name, value): def __setattr__(self, name, value):
if name not in ['symbol','store','disable_dirty'] and not self.disable_dirty: if name not in ['symbol','store','disable_dirty', 'file_offset'] and not self.disable_dirty:
self.store.dirty(self) self.store.dirty(self)
if issubclass(type(value), Base):
value = Reference.create(value)
super().__setattr__(name, value) super().__setattr__(name, value)
def __getattribute__(self, nm):
val = super().__getattribute__(nm)
if type(val) == Reference:
val = val.resolve()
return val
def update(self, d): def update(self, d):
pass pass
@ -52,22 +80,15 @@ class Base:
self.disable_dirty = True self.disable_dirty = True
self.__dict__.update(d) self.__dict__.update(d)
self.disable_dirty = False self.disable_dirty = False
def dict(self): def __getstate__(self):
r = {} r = {}
for k,v in self.__dict__.items(): for k,v in self.__dict__.items():
if k in ['store']: if k in ['store','file_offset', 'disable_dirty', 'file_offset']:
continue continue
r[k] = deepcopy(v) r[k] = deepcopy(v)
return r return r
def path(self):
raise NotImplementedError('path')
@classmethod
def ext(self):
raise NotImplementedError('extension')
def type(self): def type(self):
return self.__class__.__name__ return self.__class__.__name__

View File

@ -1,7 +1,7 @@
from .system_member import SystemMember from .base import Base
from dataclasses import field from dataclasses import field
class Jumpgate(SystemMember): class Jumpgate(Base):
def define(self): def define(self):
self.range: int = 0 self.range: int = 0
self.faction: str = '' self.faction: str = ''

View File

@ -1,10 +1,10 @@
from .system_member import SystemMember from .base import Base
from time import time from time import time
from nullptr.util import * from nullptr.util import *
from dataclasses import field from dataclasses import field
class Marketplace(SystemMember): class Marketplace(Base):
def define(self): def define(self):
self.imports:list = [] self.imports:list = []
self.exports:list = [] self.exports:list = []

View File

@ -1,10 +1,10 @@
from time import time from time import time
from nullptr.util import * from nullptr.util import *
from .system_member import SystemMember from .base import Base
size_names = ['SMALL','MODERATE','LARGE'] size_names = ['SMALL','MODERATE','LARGE']
class Survey(SystemMember): class Survey(Base):
identifier = 'signature' identifier = 'signature'
def define(self): def define(self):
self.type: str = '' self.type: str = ''

View File

@ -1,10 +0,0 @@
from .base import Base
class SystemMember(Base):
@classmethod
def ext(cls):
return 'obj'
def system(self):
p = self.symbol.split('-')
return f'{p[0]}-{p[1]}'

View File

@ -1,8 +1,8 @@
from .system_member import SystemMember from .base import Base
from nullptr.util import * from nullptr.util import *
from dataclasses import field from dataclasses import field
class Waypoint(SystemMember): class Waypoint(Base):
def define(self): def define(self):
self.x:int = 0 self.x:int = 0
self.y:int = 0 self.y:int = 0

View File

@ -1,30 +1,55 @@
from nullptr.models.base import Base from nullptr.models import *
from nullptr.models.waypoint import Waypoint
from nullptr.models.sector import Sector
from nullptr.models.system import System
from nullptr.models.agent import Agent
from nullptr.models.marketplace import Marketplace
from nullptr.models.system_member import SystemMember
from nullptr.models.jumpgate import Jumpgate
from nullptr.models.ship import Ship
from nullptr.models.contract import Contract
from nullptr.models.survey import Survey
from os.path import isfile, dirname, isdir from os.path import isfile, dirname, isdir
import os import os
from os.path import basename from os.path import basename
import json import json
from .util import * from .util import *
from time import time from time import time
import pickle
from struct import unpack, pack
class ChunkHeader:
def __init__(self):
self.in_use = True
self.size = 0
self.used = 0
@classmethod
def parse(cls, fil):
d = fil.read(16)
if len(d) < 16:
return None
# print(d)
o = cls()
d, o.used = unpack('<QQ', d)
o.size = d & 0x7fffffffffffffff
o.in_use = d & 0x8000000000000000 != 0
# print(o)
return o
def write(self, f):
d = self.size
if self.in_use:
d |= 1 << 63
d = pack('<QQ', d, self.used)
f.write(d)
def __repr__(self):
return f'chunk {self.in_use} {self.size} {self.used}'
class Store: class Store:
def __init__(self, data_dir): def __init__(self, data_file):
self.init_models() self.init_models()
self.data_dir = data_dir self.fil = open_file(data_file)
self.data = {m: {} for m in self.models} self.data = {m: {} for m in self.models}
self.system_members = {} self.system_members = {}
self.dirty_objects = set() self.dirty_objects = set()
self.cleanup_interval = 600 self.cleanup_interval = 600
self.last_cleanup = 0 self.last_cleanup = 0
self.slack = 0.1
self.slack_min = 64
self.slack_max = 1024
self.load()
def init_models(self): def init_models(self):
self.models = all_subclasses(Base) self.models = all_subclasses(Base)
@ -33,54 +58,89 @@ class Store:
def dirty(self, obj): def dirty(self, obj):
self.dirty_objects.add(obj) self.dirty_objects.add(obj)
def path(self, obj):
return os.path.join(self.data_dir, obj.path())
def load_file(self, path):
if not isfile(path):
return None
fn = basename(path)
ext = fn.split('.')[-1]
symbol = fn.split('.')[0]
if ext not in self.extensions:
return None
with open(path) as f:
data = json.load(f)
typ = self.extensions[ext]
obj = self.create(typ, symbol)
obj.load(data)
obj.store = self
return obj
def dump_object(self, obj):
return pickle.dumps(obj)
def load_object(self, data, offset):
obj = pickle.loads(data)
obj.file_offset = offset
obj.disable_dirty = False
self.hold(obj)
def load(self): def load(self):
cnt = 0 cnt = 0
start_time = time() start_time = time()
for fil in list_files(self.data_dir, True):
self.load_file(fil) self.fil.seek(0)
offset = 0
while (hdr := ChunkHeader.parse(self.fil)):
if not hdr.in_use: continue
data = self.fil.read(hdr.used)
self.load_object(data, offset)
self.fil.seek(hdr.size - hdr.used, 1)
offset = self.fil.tell()
cnt += 1 cnt += 1
dur = time() - start_time dur = time() - start_time
print(f'loaded {cnt} objects in {dur:.2f} seconds') print(f'loaded {cnt} objects in {dur:.2f} seconds')
def allocate_chunk(self, sz):
used = sz
slack = sz * self.slack
slack = min(slack, self.slack_max)
slack = max(slack, self.slack_min)
sz += slack
self.fil.seek(0, 2)
offset = self.fil.tell()
h = ChunkHeader()
h.size = sz
h.used = used
h.write(self.fil)
return offset, h
def store(self, obj): def store(self, obj):
path = self.path(obj) data = self.dump_object(obj)
path_dir = dirname(path) osize = len(data)
data = obj.dict() # is there an existing chunk for this obj?
if not isdir(path_dir): if obj.file_offset > 0:
os.makedirs(path_dir, exist_ok=True) # read chunk hdr
with open(path, 'w') as f: self.fil.seek(obj.file_offset)
json.dump(data, f, indent=2) hdr = ChunkHeader.parse(self.fil)
csize = hdr.size
def create(self, typ, symbol): # if the chunk is too small
obj = typ(symbol, self) if csize < osize:
# free the chunk
hdr.in_use = False
# force a new chunk
obj.file_offset = 0
else:
# if it is big enough, update the used field
hdr.used = osize
self.fil.seek(obj.file_offset)
hdr.write(self.fil)
if obj.file_offset == 0:
obj.file_offset, hdr = self.allocate_chunk(osize)
self.fil.write(data)
slack = b'\x00' * (hdr.size - hdr.used)
self.fil.write(slack)
def hold(self, obj):
typ = type(obj)
symbol = obj.symbol
obj.store = self
self.data[typ][symbol] = obj self.data[typ][symbol] = obj
if issubclass(typ, SystemMember): if hasattr(typ, 'system'):
system_str = obj.system() system_str = obj.system.symbol
if system_str not in self.system_members: if system_str not in self.system_members:
self.system_members[system_str] = set() self.system_members[system_str] = set()
self.system_members[system_str].add(obj) self.system_members[system_str].add(obj)
def create(self, typ, symbol):
obj = typ(symbol, self)
self.hold(obj)
self.dirty(obj)
return obj return obj
def get(self, typ, symbol, create=False): def get(self, typ, symbol, create=False):
@ -122,7 +182,7 @@ class Store:
if type(system) == System: if type(system) == System:
system = system.symbol system = system.symbol
if system not in self.system_members: if 'system' not in self.system_members:
return return
print('typ', typ) print('typ', typ)
for m in self.system_members[system]: for m in self.system_members[system]:
@ -139,9 +199,9 @@ class Store:
if o.is_expired(): if o.is_expired():
expired.append(o) expired.append(o)
for o in expired: for o in expired:
path = o.path()
if isfile(path): # TODO
os.remove(path)
del self.data[type(o)][o.symbol] del self.data[type(o)][o.symbol]
dur = time() - start_time dur = time() - start_time
# print(f'cleaned {len(expired)} in {dur:.03f} seconds') # print(f'cleaned {len(expired)} in {dur:.03f} seconds')
@ -153,6 +213,7 @@ class Store:
for obj in self.dirty_objects: for obj in self.dirty_objects:
it += 1 it += 1
self.store(obj) self.store(obj)
self.fil.flush()
self.dirty_objects = set() self.dirty_objects = set()
dur = time() - start_time dur = time() - start_time
# print(f'flush done {it} items {dur:.2f}') # print(f'flush done {it} items {dur:.2f}')

View File

@ -1,21 +1,16 @@
from datetime import datetime from datetime import datetime
from math import ceil from math import ceil
import os import os
from os.path import isfile from os.path import isfile, dirname
def list_files(path, recursive=False): def open_file(fn):
if recursive: d = dirname(fn)
for p, dirnames, fils in os.walk(path): os.makedirs(d, exist_ok=True)
for f in fils: if isfile(fn):
fil = os.path.join(p, f) return open(fn, 'rb+')
yield fil
else: else:
for f in os.listdir(path): return open(fn, 'ab+')
fil = os.path.join(path, f)
if not isfile(fil):
continue
yield fil
def must_get(d, k): def must_get(d, k):
if type(k) == str: if type(k) == str:
k = k.split('.') k = k.split('.')

View File

@ -3,6 +3,10 @@ This project uses a custom database format just because we can.
The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk. The script reads the entire store on startup. Each object can br altered in memory. A 'dirty' status is set when an object is changed. periodically the script will flush the store, writing all changes back to disk.
The store disk format is optimized for two things:
* Loading the entire contents in memory
* Writing changed objects back to disk
## Objects ## Objects
First lets discuss what we are storing. First lets discuss what we are storing.
@ -30,21 +34,17 @@ An index is a dict with a string as key and a list of objects as value. The dict
* store.load(fil) loads all objects * store.load(fil) loads all objects
* store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present * store.get(type, symbol, create=False) fetches the object. If create==False: None if it wasnt present
* store.all(type) generator for all objects of a goven type * store.all(type) generator for all objects of a goven type
* store.delete(typ, symbol)
* store.cleanup() removes all expired objects * store.cleanup() removes all expired objects
* store.flush() writes all dirty objects to disk * store.flush() writes all dirty objects to disk
* * store.defrag() consolidates the store file to minimize storage and loading time
type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base type may be a class or a string containing the name of a class. The type should be a subclass of models.base.Base
# file format # file format
Until specified otherwise, all numbers are stored low-endian 64bit unsigned. Until specified otherwise, all numbers are stored low-endian 64bit unsigned.
the file format is a header followed by a number of blocks. the size and number of blocks are dictated by the header: The store file is built up out of chunks. A chunk is either empty or houses exactly one file. If a file is updated and its size fits the chunk, it is updated in-place. If the new content does not fit the chunk, a new chunk is allocated at the end of the file. The old chunk is marked as empty.
* Magic A chunk starts with a chunk header. This is just a single field describing the size of the chunk in bytes, not including the header. The first bit of the field is the IN_USE flag. If it is not set, the contents of the chunk are ignored during loading.
* Blocksize in bytes
* Number of blocks
* Root file
* Free file
A block is prefixed by a pointer to the next block of that file. In the last block of the file, the pointer is 0. So if you have 1000 byte blocks, each block takes 1008 bytes of space.