Browse Source

partially implemented new api into the frontend

automate2
Stephen Lorenz 4 years ago
parent
commit
4b483ff0d1
  1. BIN
      cjs/.app/database/debug/database.db
  2. BIN
      cjs/.app/state.db
  3. 0
      cjs/.resources/mailer.json
  4. 0
      cjs/.resources/settings.json
  5. 0
      cjs/.resources/tutorialdata/FreqDatabases/ESX17_Norway.csv
  6. 0
      cjs/.resources/tutorialdata/FreqDatabases/Fusion 6C_Norway.csv
  7. 0
      cjs/.resources/tutorialdata/FreqDatabases/Identifiler_Caucasian.csv
  8. 0
      cjs/.resources/tutorialdata/FreqDatabases/Identifiler_NIST.csv
  9. 0
      cjs/.resources/tutorialdata/FreqDatabases/NGM_Holland.csv
  10. 0
      cjs/.resources/tutorialdata/FreqDatabases/SGMPlus_Norway.csv
  11. 0
      cjs/.resources/tutorialdata/FreqDatabases/SGMPlus_UK.csv
  12. 0
      cjs/.resources/tutorialdata/databaseESX17.txt
  13. 0
      cjs/.resources/tutorialdata/my_project_file.Rdata
  14. 0
      cjs/.resources/tutorialdata/refs.csv
  15. 0
      cjs/.resources/tutorialdata/stain.txt
  16. 22
      cjs/cjs.py
  17. 357
      cjs/core/database.py
  18. 2
      cjs/core/state.py
  19. 157
      cjs/database.py
  20. 50
      cjs/debug.py
  21. 8
      cjs/models/database.py
  22. 3
      cjs/utils/cli.py
  23. 15
      cjs/utils/fs.py

BIN
cjs/.data/my_database/database.db → cjs/.app/database/debug/database.db

BIN
cjs/.data/state.db → cjs/.app/state.db

0
cjs/resources/mailer.json → cjs/.resources/mailer.json

0
cjs/resources/settings.json → cjs/.resources/settings.json

0
cjs/resources/tutorialdata/FreqDatabases/ESX17_Norway.csv → cjs/.resources/tutorialdata/FreqDatabases/ESX17_Norway.csv

0
cjs/resources/tutorialdata/FreqDatabases/Fusion 6C_Norway.csv → cjs/.resources/tutorialdata/FreqDatabases/Fusion 6C_Norway.csv

0
cjs/resources/tutorialdata/FreqDatabases/Identifiler_Caucasian.csv → cjs/.resources/tutorialdata/FreqDatabases/Identifiler_Caucasian.csv

0
cjs/resources/tutorialdata/FreqDatabases/Identifiler_NIST.csv → cjs/.resources/tutorialdata/FreqDatabases/Identifiler_NIST.csv

0
cjs/resources/tutorialdata/FreqDatabases/NGM_Holland.csv → cjs/.resources/tutorialdata/FreqDatabases/NGM_Holland.csv

0
cjs/resources/tutorialdata/FreqDatabases/SGMPlus_Norway.csv → cjs/.resources/tutorialdata/FreqDatabases/SGMPlus_Norway.csv

0
cjs/resources/tutorialdata/FreqDatabases/SGMPlus_UK.csv → cjs/.resources/tutorialdata/FreqDatabases/SGMPlus_UK.csv

0
cjs/resources/tutorialdata/databaseESX17.txt → cjs/.resources/tutorialdata/databaseESX17.txt

0
cjs/resources/tutorialdata/my_project_file.Rdata → cjs/.resources/tutorialdata/my_project_file.Rdata

0
cjs/resources/tutorialdata/refs.csv → cjs/.resources/tutorialdata/refs.csv

0
cjs/resources/tutorialdata/stain.txt → cjs/.resources/tutorialdata/stain.txt

22
cjs/cjs.py

@ -17,10 +17,19 @@ _commands = [
# main-entry point to the entire command-line interface
@click.group()
@click.option('-v', '--verbose', help='Print additional information.',
@click.option('--data_dir',
help='Set application data location.',
type=click.Path(),
default='.app')
@click.option('--resource_dir',
help='Set resource data location.',
type=click.Path(),
default='.resources')
@click.option('-v', '--verbose',
help='Print additional information.',
is_flag=True)
@click.pass_context
def cli(ctx, verbose):
def cli(ctx, data_dir, resource_dir, verbose):
"""
A command-line interface wrapper to the Criminal Justice Software toolchain.\f
Args:
@ -28,14 +37,17 @@ def cli(ctx, verbose):
verbose: Flag to enable additional prints.
database: Path to a SQLite3 database.
"""
ctx.ensure_object(dict)
# determine project working directory
project_dir = '/home/csguest/Desktop/cjs2/cjs'
project_path = lambda path: '%s/%s' % (project_dir, path)
# initialize context object
# store non-click information
# TODO: replace hard coded path with a more cross-platform method
ctx.obj['data_dir'] = '/home/csguest/Desktop/cjsoftware/cjs/.data'
ctx.obj['resource_dir'] = '/home/csguest/Dekstop/cjsoftware/cjs/resources'
ctx.obj['data_dir'] = project_path(data_dir)
ctx.obj['resource_dir'] = project_path(resource_dir)
# store click-related information
# global flags

357
cjs/core/database.py

@ -1,207 +1,257 @@
#!/usr/bin/env python3
# external
# standard library modules
import functools
from functools import wraps
# pip modules
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy.exc import IntegrityError
# local modules
import core.state
import utils.fs
from utils.fs import remove_file, abs_path, copy_file, create_dir
import utils.io
from utils.io import write_json, get_valid_filename
from core.state import add_state
from core.state import remove_state
from core.state import select_state
import models.state
from models.state import StateVault
from models.state import DatabaseState
import models.database
#from models.database import DatabaseBase
from models.database import *
# BEGIN vault backend
# BEGIN: Generic Database API
# modified from https://stackoverflow.com/a/36944992
# NOTE: ideally this would be inside the database class
def _pass_session(fn):
@wraps(fn) # get around the fact a decorator cannot have self
def wrapper(self, *fn_args, **fn_kwargs):
s = self._get_session()
res = fn(self, s, *fn_args, **fn_kwargs)
s.close()
return res
return wrapper
class Database:
def __init__(self, path):
self._engine = create_engine('sqlite:///%s' % path)
def __init__(self, location):
self.location = location # working directory of the database
self._engine = create_engine('sqlite:///%s/database.db' % location)
self._SessionFactory = sessionmaker(bind=self._engine)
def _get_session(self):
DatabaseBase.metadata.create_all(self._engine)
return self._SessionFactory
def get_session(func):
def wrapper(self, *args, **kwargs):
s = self._get_session()
res = func(s, *args, **kwargs)
s.close()
return res
return wrapper
@pass_session
def add_table(self, sess, Table):
# TODO: make insert one or many
sess.add(Table)
sess.commit()
@pass_session
def remove_table(sess, Table, expr):
sess.query(Table).filter(expr).delete()
sess.commit()
@pass_session
return self._SessionFactory()
# TODO: replace this function with a better method of doing this
@_pass_session
def initialize(self, sess):
# NOTE: this method was created because I was unable to get sqlalchemy's
# initial values to work
names = ['Pending', 'Processing', 'Complete', 'Failed']
events = [EventTable(n) for n in names]
self.add_tables(events)
@_pass_session
def add_tables(self, sess, Tables):
try:
for T in Tables:
sess.add(T)
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def remove_table(sess, Table, expr=True):
try:
sess.query(Table).filter(expr).delete()
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def update_table(sess, Table, expr):
pass
@pass_session
def select_table(self, sess, Table, expr):
q = s.query(Table).filter(expr).all()
return q
@pass_session
def clear_table(self, sess, Table):
s.query(Table).delete()
s.commit()
def get_tests(self):
return self._get(VaultTest)
def clear_jobs(self):
self._clear(VaultJob)
def insert_evidence(self, name, path):
evidence = VaultEvidence(name, path)
self._insert(evidence)
return evidence.id
def insert_comparison(self, name, path):
comparison = VaultComparison(name, path)
self._insert(comparison)
return comparison.id
def insert_test(self, evidence_id, comparison_id):
test = VaultTest(evidence_id, comparison_id)
self._insert(test)
return test.id
def insert_event(self, name):
event = VaultEvent(name)
self._insert(event)
return event.id
def insert_job(self, test_id, batch_id):
job = VaultJob(test_id, batch_id)
self._insert(job)
return job.id
def insert_batch(self, name):
batch = VaultBatch(name)
self._insert(batch)
return batch.id
def insert_result(self, name, path):
result = VaultResult(name, path)
self._insert(result)
return result
# new api
def _insert(self, Table):
s = self.get_session()
@_pass_session
def select_tables(self, sess, Table, expr=True):
try:
res = []
for t in Table:
s.add(t)
s.flush()
s.refresh(t)
res.append(t.id)
q = sess.query(Table).filter(expr).all()
return q
except:
s.add(Table)
s.flush()
s.refresh(Table)
res = Table.id
s.commit()
s.close()
return res
# TODO: implement fine-grained handling
raise
def insert(self, Table):
return self._insert(Table)
# END: Generic Database API
def _query(self, Table):
s = self.get_session()
q = s.query(Table)
s.close()
return q
# BEGIN: Internal functions
# BEGIN: Database State functions
def _get_state(name):
try:
ds = select_state(DatabaseState,
DatabaseState.name == name)
return ds[0] # select_state returns a list
except:
# TODO: implement fine-grained handling
raise
def sizeof(self, Table, where=None):
if where:
return self._query(Table).filter(where).count()
else:
return self._query(Table).count()
# END: Database State functions
def next(self, Table):
s = self.get_session()
q = s._query(Table).filter(event_id=1)
# BEGIN: Database object functions
def _get_database(name):
try:
ds = _get_state(name)
db = Database(ds.location)
return db
except:
# TODO: implement fine-grained handling
raise
# TODO: name is easily confused with add_database, consider refactoring
def _database_add(name, Tables):
try:
db = _get_database(name)
db.add_tables(Tables)
except:
# TODO: implement fine-grained handling
raise
# TODO: name is easily confused with remove_database, consider refactoring
def _database_remove(name, Table):
try:
db = _get_database(name)
# remove_table's expr parameter is true by default
# and will remove everything in the given table if
# not passed another value
db.remove_table(Table)
except:
# TODO: implement fine-grained handling
raise
def select_all(self, Table):
return self._query(Table).all()
# TODO: name is easily confused with add_database, consider refactoring
def _database_update(name, Table):
try:
db = _get_database(name)
pass
except:
# TODO: implement fine-grained handling
raise
def select_first(self, Table):
return self._query(Table).first()
# TODO: name is easily confused with add_database, consider refactoring
def _database_select(name, Table, expr):
try:
db = _get_database(name)
Ts = db.select_tables(Table, expr) # returns list of Tables
return Ts
except:
# TODO: implement fine-grained handling
raise
def select_last(self, Table):
return self._query(Table).order_by(Table.id.desc()).first()
# END: Database object functions
# END: Internal functions
# public functions
# BEGIN: External functions
# BEGIN: Database State functions
def new(database_name, database_root):
def add_database(name, location, init=True):
try:
path = f'{database_root}/database.db'
s = StateVault(database_name, database_root, path)
core.state.add_state(s)
ds = DatabaseState(name, location)
add_state(ds)
# TODO: replace with a better method of doing this
if init:
db = _get_database(name)
db.initialize()
except:
# TODO: implement fine-grained handling
raise
def list_databases():
def remove_database(name):
try:
databases = core.state.list_state(StateVault)
return databases
remove_state(DatabaseState,
DatabaseState.name == name)
except:
# TODO: implement fine-grained handling
raise
def delete(name):
vault = _state_get_by_name(name)
def update_database(name):
try:
remove_file(vault.root)
except :
pass
_state_remove_by_name(name)
except:
# TODO: implement fine-grained handling
raise
def list_database():
try:
# select_state's expr parameter is true by default
# and will return everything in the state table if
# not passed another value
ds = select_state(DatabaseState)
return ds
except:
# TODO: implement fine-grained handling
raise
# END: Database State functions
# BEGIN: Database object functions
def stage(name, batch):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
def database_location(name):
try:
ds = _get_database(name)
return ds.location
except:
raise
batch_id = vault.insert(VaultBatch(batch))
# END: Database object functions
tests = vault.get_tests()
l = [VaultJob(t.id, batch_id) for t in tests]
vault.insert(l)
# END: External functions
def tester():
name = 'debug'
#location = '.data/my_database'
#add_database(name, location)
#print(list_database())
#database_add(name, [EvidenceTable('my_file', '/path/to/file')])
print(_database_select(name, EventTable, EventTable.name == 'Pending')[0].name)
#tester()
'''
def upload(name, evidence, comparisons):
vault_state = _state_get_by_name(name)
vault_root = vault_state.root
vault = Vault(vault_state.path)
try:
db = _get_database(name)
# format
except:
# TODO: implement fine-grained handling
raise
evidence_name = evidence['name']
evidence_file = '%s/input/%s.json' % (vault_root, evidence_name)
evidence_file = '%s/input/%s.json' % (db.location, evidence_name)
evidence_file = str(abs_path(evidence_file))
write_json(evidence_file, evidence)
evidence_id = vault.insert(VaultEvidence(evidence_name, evidence_file))
'''
'''
def stage(name, batch):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
batch_id = vault.insert(VaultBatch(batch))
tests = vault.get_tests()
l = [VaultJob(t.id, batch_id) for t in tests]
vault.insert(l)
for c in comparisons:
comparison_name = c['name']
@ -213,20 +263,6 @@ def upload(name, evidence, comparisons):
vault.insert(VaultTest(evidence_id, comparison_id))
def clear(name):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
vault.clear_jobs()
def vault_path(name):
v = _state_get_by_name(name)
return v.path
def vault_name(path):
v = _state_get_by_path(path)
return v.name
def next_job(name):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
@ -282,5 +318,4 @@ def status(name):
status['next_job'] = [evidence_name, comparison_name]
s.close()
return status
'''

2
cjs/core/state.py

@ -11,7 +11,7 @@ import models.state
from models.state import StateBase
# TODO: replace with project relative path
_engine = create_engine('sqlite:///.data/state.db')
_engine = create_engine('sqlite:///.app/state.db')
_SessionFactory = sessionmaker(bind=_engine)

157
cjs/database.py

@ -1,25 +1,42 @@
#!/usr/bin/env python3
# external
# pip modules
import click
# local modules
import utils.cli
from utils.cli import echo_title
from utils.cli import echo_field
from utils.cli import echo_enum
import utils.io
from utils.io import read_json, get_valid_filename
from utils.io import read_json
from utils.io import get_valid_filename
import utils.fs
from utils.fs import remove_file, file_exists
import utils.cli
from utils.cli import echo_title, echo_field
from utils.fs import create_dir
from utils.fs import remove_dir
from utils.fs import remove_file
from utils.fs import file_exists
from utils.fs import copy_dir
import core.database
from core.database import add_database
from core.database import remove_database
from core.database import update_database # TODO: unused
from core.database import list_database
from core.database import database_location
def init_database_skel(root):
create_dir(root)
create_dir(f'{root}/input')
create_dir(f'{root}/output')
create_dir(f'{root}/log')
def database_skeleton(root):
try:
create_dir(root)
create_dir('%s/input' % root)
create_dir('%s/output' % root)
create_dir('%s/tmp' % root)
create_dir('%s/log' % root)
except Exception as e:
print('Error: Unable to initialize database skeleton: %s' % e)
raise
# BEGIN command-line interface
@ -33,13 +50,15 @@ def database(ctx):
ctx: Click command-line interface context.
"""
# ensure database directory exists
database_dir = '%s/database' % ctx.obj['data_dir']
utils.fs.create_dir(database_dir)
# store database_dir to context object
ctx.obj['database_dir'] = database_dir
# ensure database directory exists
create_dir(database_dir)
@database.command()
@click.argument('database_name')
@click.pass_context
@ -49,40 +68,52 @@ def new(ctx, database_name):
Args:
ctx: Click command-line interface context.
database_name: Name used to register a new database.
'''
try:
# try to register the new database
core.database.new(database_name)
#new_dir = f'{ctx.obj['database_dir']}/{get_valid_filename(database_name)}'
#init_database_skel(new_dir)
# determine where the database will be stored
database_root = get_valid_filename(database_name)
new_location = '%s/%s' % (ctx.obj['database_dir'],
database_root)
database_skeleton(new_location)
# register the new database to the created directory
add_database(database_name, new_location)
except Exception as e:
# TODO: better exception handling
click.echo("Error: Unable to create '%s': %s" % (db, e))
ctx.exit()
# if database was created successfully, initialize
click.echo("Error: Unable to create new database: %s" % e)
raise
@database.command(name='list')
@database.command()
@click.argument('database_name')
@click.argument('new_database')
@click.pass_context
def list_databases(ctx):
def clone(ctx, database_name, new_database):
'''
Display available databases.\f
Args:
ctx: Click command-line interface context.
Copy a database.\f
'''
echo_title('Available Databases', fg='yellow', bold=True)
databases = core.database.list_databases()
for i, db in enumerate(databases, start=1):
echo_field(db.name, db.root, fg='yellow', bold=True)
try:
# get a valid directory name for the database
database_root = get_valid_filename(new_database)
# determine where the database will be stored
new_location = '%s/%s' % (ctx.obj['database_dir'], database_root)
# get the location of the pre-existing database
old_location = database_location(database_name)
# copy it's contents recursively to the new destination
copy_dir(old_location, new_location)
# register the new database to the created directory
add_database(new_database, new_location, init=False)
except Exception as e:
click.echo(e)
raise
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
@database.command()
@click.argument('database_name')
@click.pass_context
def delete(ctx, db):
def delete(ctx, database_name):
'''
Remove a database.\f
@ -91,15 +122,36 @@ def delete(ctx, db):
src: Source directory containing input file(s)
'''
if click.confirm("Warning: This action cannot be undone!\nDelete database? (%s)" % database):
if click.confirm("Warning: This action cannot be undone!\nDelete database?"):
try:
core.database.delete(db)
except:
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
# delete/clean up it's data directory
location = database_location(database_name)
remove_dir(location)
# unregister database
remove_database(database_name)
click.echo('Database deleted successfully.')
except Exception as e:
click.echo('Error: Unable to delete database: %s' % e)
raise
else:
click.echo('No changes were made.')
click.echo('database successfully deleted.')
@database.command(name='list')
@click.pass_context
def my_list(ctx):
'''
Display available databases.\f
Args:
ctx: Click command-line interface context.
'''
echo_title('Available Databases', fg='yellow', bold=True)
databases = list_database()
for i, db in enumerate(databases, start=1):
echo_enum(i, db.name, fg='yellow')
@database.command()
@click.argument('database')
@ -197,28 +249,5 @@ def status(db):
click.echo('No job currently staged.')
@database.command()
@click.argument('database')
@click.pass_context
def clone(ctx, db):
'''
Copy a database.\f
'''
click.echo('New database:')
name = click.prompt("Name (e.g. 'my_database')", type=str)
while True:
path = click.prompt("Location (e.g. 'path/to/database.db')", type=str)
if not file_exists(path):
break
click.echo("'%s' already exists." % path)
try:
core.database.copy(db, path)
core.database.new(name, path)
except Exception as e:
click.echo(e)
raise
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
# END command-line interface

50
cjs/debug.py

@ -38,6 +38,8 @@ def _pass_session(fn):
class Database:
def __init__(self, location):
self.location = location # working directory of the database
self._engine = create_engine('sqlite:///%s/database.db' % location)
self._SessionFactory = sessionmaker(bind=self._engine)
@ -89,12 +91,24 @@ class Database:
# END: Generic Database API
# BEGIN: Internal functions
# BEGIN: Database object functions
def _get_database(name):
# BEGIN: Database State functions
def _get_state(name):
try:
ds = select_state(DatabaseState,
DatabaseState.name == name)
db = Database(ds[0].location) # select_state returns a list
return ds[0] # select_state returns a list
except:
# TODO: implement fine-grained handling
raise
# END: Database State functions
# BEGIN: Database object functions
def _get_database(name):
try:
ds = _get_state(name)
db = Database(ds.location)
return db
except:
# TODO: implement fine-grained handling
@ -188,7 +202,6 @@ def list_database():
# END: External functions
def tester():
name = 'debug'
#location = '.data/my_database'
@ -197,8 +210,24 @@ def tester():
#database_add(name, [EvidenceTable('my_file', '/path/to/file')])
print(_database_select(name, EventTable, EventTable.name == 'Pending')[0].name)
tester()
#tester()
'''
def upload(name, evidence, comparisons):
try:
db = _get_database(name)
# format
except:
# TODO: implement fine-grained handling
raise
evidence_name = evidence['name']
evidence_file = '%s/input/%s.json' % (db.location, evidence_name)
evidence_file = str(abs_path(evidence_file))
write_json(evidence_file, evidence)
evidence_id = vault.insert(VaultEvidence(evidence_name, evidence_file))
'''
'''
def stage(name, batch):
vault_state = _state_get_by_name(name)
@ -210,18 +239,7 @@ def stage(name, batch):
l = [VaultJob(t.id, batch_id) for t in tests]
vault.insert(l)
def upload(name, evidence, comparisons):
vault_state = _state_get_by_name(name)
vault_root = vault_state.root
vault = Vault(vault_state.path)
evidence_name = evidence['name']
evidence_file = '%s/input/%s.json' % (vault_root, evidence_name)
evidence_file = str(abs_path(evidence_file))
write_json(evidence_file, evidence)
evidence_id = vault.insert(VaultEvidence(evidence_name, evidence_file))
for c in comparisons:
comparison_name = c['name']

8
cjs/models/database.py

@ -23,7 +23,7 @@ class EvidenceTable(DatabaseBase):
def __str__(self):
return '%s, %s, %s' % (self.id, self.name, self.path)
class VaultComparison(DatabaseBase):
class ComparisonTable(DatabaseBase):
'''Used to store comparison files.'''
__tablename__ = 'Comparison'
@ -39,7 +39,7 @@ class VaultComparison(DatabaseBase):
def __str__(self):
return '%s, %s, %s' % (self.id, self.name, self.path)
class VaultTest(DatabaseBase):
class TestTable(DatabaseBase):
'''Used to store comparison files.'''
__tablename__ = 'Test'
@ -84,6 +84,10 @@ class JobTable(DatabaseBase):
self.test_id = test_id
self.batch_id = batch_id
def __str__(self):
return '%s, %s, %s, %s, %s' % (self.id, self.test_id, self.event_id,
self.batch_id, self.timestamp)
class BatchTable(DatabaseBase):
'''Used to identify which set that a result belongs to.'''

3
cjs/utils/cli.py

@ -10,3 +10,6 @@ def echo_title(title, *args, **kwargs):
def echo_field(label, value, *args, **kwargs):
# TODO: add dynamic indent
echo(style(f' {label}: ', *args, **kwargs) + value)
def echo_enum(i, value, *args, **kwargs):
echo(f' {i}. ' + style(f'{value}', *args, **kwargs))

15
cjs/utils/fs.py

@ -5,12 +5,16 @@ from pathlib import Path
import shutil
def _copy(self, target):
def _copy_file(self, target):
assert self.is_file()
shutil.copy(self, target)
pathlib.Path.copy = _copy
def _copy_dir(self, target):
assert self.is_dir()
shutil.copytree(self, target)
pathlib.Path.copy_file = _copy_file
pathlib.Path.copy_dir = _copy_dir
def project_path(path):
'''Given a path, return relative to the script.'''
@ -47,4 +51,7 @@ def remove_file(filename):
Path(filename).unlink()
def copy_file(source, destination):
Path(source).copy(destination)
Path(source).copy_file(destination)
def copy_dir(source, destination):
Path(source).copy_dir(destination)
Loading…
Cancel
Save