Browse Source

go back to lab

cjs3
Stephen Lorenz 3 years ago
parent
commit
fcf67653f4
  1. 274
      cjs/database.py
  2. 64
      cjsd/cjs.py
  3. 3
      cjsd/cjsd.py
  4. 0
      cjsd/resources/__init__.py
  5. 275
      cjsd/resources/database.py
  6. 2
      run_cjsd

274
cjs/database.py

@ -0,0 +1,274 @@
@click.group()
@click.pass_context
def database(ctx):
"""
Manage automation databases.\f
Args:
ctx: Click command-line interface context.
"""
# store database_dir to context object
ctx.obj['database_dir'] = database_dir
# ensure database directory exists
create_dir(database_dir)
@database.command()
@click.argument('database_name')
@click.pass_context
def new(ctx, database_name):
'''
Create a new database.\f
Args:
ctx: Click command-line interface context.
database_name: Name used to register a new database.
'''
@database.command()
@click.argument('database_name')
@click.argument('new_database')
@click.pass_context
def clone(ctx, database_name, new_database):
'''
Copy a database.\f
'''
try:
# get a valid directory name for the database
database_root = get_valid_filename(new_database)
# determine where the database will be stored
new_location = '%s/%s' % (ctx.obj['database_dir'], database_root)
# get the location of the pre-existing database
old_location = database_location(database_name)
# copy it's contents recursively to the new destination
copy_dir(old_location, new_location)
# register the new database to the created directory
add_database(new_database, new_location, init=False)
except Exception as e:
click.echo(e)
raise
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
@database.command()
@click.argument('database_name')
@click.pass_context
def delete(ctx, database_name):
'''
Remove a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
if click.confirm("Warning: This action cannot be undone!\nDelete database?"):
try:
# delete/clean up it's data directory
location = database_location(database_name)
remove_dir(location)
# unregister database
remove_database(database_name)
click.echo('Database deleted successfully.')
except Exception as e:
click.echo('Error: Unable to delete database: %s' % e)
raise
else:
click.echo('No changes were made.')
@database.command(name='list')
@click.pass_context
def my_list(ctx):
'''
Display available databases.\f
Args:
ctx: Click command-line interface context.
'''
echo_title('Available Databases', fg='yellow', bold=True)
databases = list_database()
for i, db in enumerate(databases, start=1):
echo_enum(i, db.name, fg='yellow')
@database.command()
@click.argument('database_name')
@click.argument('input_file')
@click.pass_context
def upload(ctx, database_name, input_file):
'''
Add data to a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
click.echo('Please wait, this may take awhile...')
try:
# get the location of the database
location = database_location(database_name)
input_path = lambda root, filename: '%s/input/%s.json' % (root, filename)
# read the input data into a dictionary
input_data = read_json(input_file)
click.echo('Validating input...')
database_data = []
non_contrib = []
with click.progressbar(input_data.items()) as bar:
for key, value in bar:
tmp_data = []
# TODO: implement models.reqbt handler for validation
try:
# create a new copy of the current data
data = dict(value)
# try to see if it's a known contributor
# separate the comparison(s)
comparisons = data.pop('comparisons')
evidence = data # rest of the data
# TODO: clean this up
evidence_name = evidence['name']
evidence_file = input_path(location, evidence_name)
write_json(evidence_file, evidence) # save file
tmp_comparison = []
for comparison in comparisons:
comparison_name = comparison['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison) # save file
tmp_comparison.append(ComparisonTable(comparison_name, comparison_file))
database_data.append([EvidenceTable(evidence_name, evidence_file), tmp_comparison])
# try to see if it's a non-contributors
except ValueError:
data = list(value)
for comparison_dict in data:
comparison_name = comparison_dict['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison_dict) # save file
non_contrib.append(ComparisonTable(comparison_name, comparison_file))
click.echo('Storing data...')
if non_contrib:
evidence_list = _database_select(database_name, EvidenceTable)
database_data = []
for evidence_table in evidence_list:
database_data.append([evidence_table, non_contrib])
_database_compound_add(database_name, TestTable, database_data)
else:
_database_compound_add(database_name, TestTable, database_data)
click.echo('Data successfully uploaded.')
except:
click.echo("Error: Unable to read '%s'." % input_file)
raise
@database.command()
@click.argument('database_name')
@click.pass_context
def clear(ctx, database_name):
'''
Clear jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: This action cannot be undone!\nClear the Job table?"):
try:
_database_remove(database_name, JobTable)
except Exception as e:
click.echo("Error: Unable to clear jobs.")
raise
click.echo('Job table successfully cleared.')
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database_name')
@click.argument('batch_name', type=str)
@click.pass_context
def stage(ctx, database_name, batch_name):
'''
Generate jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: There are still jobs currently pending.\nLoad the Job table?"):
try:
location = database_location(database_name)
create_dir('%s/output/%s' % (location, batch_name))
_database_add(database_name, [BatchTable(batch_name)])
batch = _database_select(database_name,
BatchTable,
BatchTable.name == batch_name)
batch = batch[0] # _database_select returns a list, but this is a single value
tests = _database_select(database_name, TestTable)
_database_add(database_name, [JobTable(t.id_, batch.id_) for t in tests])
except Exception as e:
raise
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database_name')
def status(database_name):
'''
Display information about a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
get_count = lambda Table, expr=True: database_count(database_name, Table, expr)
evidence_count = get_count(EvidenceTable)
comparison_count = get_count(ComparisonTable)
test_count = get_count(TestTable)
job_count = get_count(JobTable)
pending_count = get_count(JobTable, JobTable.event_id == 1)
processing_count = get_count(JobTable, JobTable.event_id == 2)
complete_count = get_count(JobTable, JobTable.event_id == 3)
failed_count = get_count(JobTable, JobTable.event_id == 4)
evidence, comparison = next_job(database_name)
job = '%s and %s' % (evidence.name, comparison.name)
batch = current_batch(database_name).name
echo_title('Database Status', fg='yellow', bold=True)
echo_field('Evidence', evidence_count, fg='yellow')
echo_field('Comparison', comparison_count, fg='yellow')
echo_field('Tests', test_count, fg='yellow')
click.echo('')
echo_field('Jobs', job_count, fg='yellow')
echo_field(' Pending', pending_count, fg='yellow')
echo_field(' Processing', processing_count, fg='yellow')
echo_field(' Complete', complete_count, fg='yellow')
echo_field(' Failed', failed_count, fg='yellow')
click.echo('')
echo_field('Current Batch', batch, fg='yellow')
echo_field('Next Job', job, fg='yellow')
echo_field('Average Time', '23.3 sec', fg='yellow')

64
cjsd/cjs.py

@ -1,64 +0,0 @@
#!/usr/bin/env python3
import click
# TODO: replace with plugin manager
from database import database
from automate import automate
from reqbt import reqbt
from graphs import graphs
from convert import convert
_commands = [
database,
automate,
reqbt,
graphs,
convert
]
# main-entry point to the entire command-line interface
@click.group()
@click.option('-v', '--verbose',
help='Print additional information.',
is_flag=True)
@click.option('--data_dir',
help='Set the application data directory.',
type=click.Path(),
default='.app')
@click.option('--resource_dir',
help='Set the resource data directory.',
type=click.Path(),
default='resources')
@click.pass_context
def cli(ctx, data_dir, resource_dir, verbose):
"""
A command-line interface wrapper to the Criminal Justice Software toolchain.\f
Args:
ctx: Click command-line interface context.
verbose: Flag to enable additional prints.
database: Path to a SQLite3 database.
"""
ctx.ensure_object(dict)
# determine project working directory
project_dir = '/home/csguest/Desktop/cjs2/cjs' # '/home/dev/cjs2/cjs'
project_path = lambda path: '%s/%s' % (project_dir, path)
# initialize context object
# store non-click information
# TODO: replace hard coded path with a more cross-platform method
ctx.obj['data_dir'] = project_path(data_dir)
ctx.obj['resource_dir'] = project_path(resource_dir)
# store click-related information
# global flags
ctx.obj['verbose'] = verbose
# load external modules into cli
for cmd in _commands:
cli.add_command(cmd)
if __name__ == '__main__':
cli()

3
cjsd/app.py → cjsd/cjsd.py

@ -6,7 +6,8 @@ import json
import falcon
from falcon import API
import database
import resources
from resources import database
PKG_DIR = os.path.dirname(os.path.abspath(__file__))

0
cjsd/resources/__init__.py

275
cjsd/database.py → cjsd/resources/database.py

@ -286,279 +286,4 @@ class StageResource:
else:
click.echo('No changes were made.')
@click.group()
@click.pass_context
def database(ctx):
"""
Manage automation databases.\f
Args:
ctx: Click command-line interface context.
"""
# store database_dir to context object
ctx.obj['database_dir'] = database_dir
# ensure database directory exists
create_dir(database_dir)
@database.command()
@click.argument('database_name')
@click.pass_context
def new(ctx, database_name):
'''
Create a new database.\f
Args:
ctx: Click command-line interface context.
database_name: Name used to register a new database.
'''
@database.command()
@click.argument('database_name')
@click.argument('new_database')
@click.pass_context
def clone(ctx, database_name, new_database):
'''
Copy a database.\f
'''
try:
# get a valid directory name for the database
database_root = get_valid_filename(new_database)
# determine where the database will be stored
new_location = '%s/%s' % (ctx.obj['database_dir'], database_root)
# get the location of the pre-existing database
old_location = database_location(database_name)
# copy it's contents recursively to the new destination
copy_dir(old_location, new_location)
# register the new database to the created directory
add_database(new_database, new_location, init=False)
except Exception as e:
click.echo(e)
raise
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
@database.command()
@click.argument('database_name')
@click.pass_context
def delete(ctx, database_name):
'''
Remove a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
if click.confirm("Warning: This action cannot be undone!\nDelete database?"):
try:
# delete/clean up it's data directory
location = database_location(database_name)
remove_dir(location)
# unregister database
remove_database(database_name)
click.echo('Database deleted successfully.')
except Exception as e:
click.echo('Error: Unable to delete database: %s' % e)
raise
else:
click.echo('No changes were made.')
@database.command(name='list')
@click.pass_context
def my_list(ctx):
'''
Display available databases.\f
Args:
ctx: Click command-line interface context.
'''
echo_title('Available Databases', fg='yellow', bold=True)
databases = list_database()
for i, db in enumerate(databases, start=1):
echo_enum(i, db.name, fg='yellow')
@database.command()
@click.argument('database_name')
@click.argument('input_file')
@click.pass_context
def upload(ctx, database_name, input_file):
'''
Add data to a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
click.echo('Please wait, this may take awhile...')
try:
# get the location of the database
location = database_location(database_name)
input_path = lambda root, filename: '%s/input/%s.json' % (root, filename)
# read the input data into a dictionary
input_data = read_json(input_file)
click.echo('Validating input...')
database_data = []
non_contrib = []
with click.progressbar(input_data.items()) as bar:
for key, value in bar:
tmp_data = []
# TODO: implement models.reqbt handler for validation
try:
# create a new copy of the current data
data = dict(value)
# try to see if it's a known contributor
# separate the comparison(s)
comparisons = data.pop('comparisons')
evidence = data # rest of the data
# TODO: clean this up
evidence_name = evidence['name']
evidence_file = input_path(location, evidence_name)
write_json(evidence_file, evidence) # save file
tmp_comparison = []
for comparison in comparisons:
comparison_name = comparison['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison) # save file
tmp_comparison.append(ComparisonTable(comparison_name, comparison_file))
database_data.append([EvidenceTable(evidence_name, evidence_file), tmp_comparison])
# try to see if it's a non-contributors
except ValueError:
data = list(value)
for comparison_dict in data:
comparison_name = comparison_dict['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison_dict) # save file
non_contrib.append(ComparisonTable(comparison_name, comparison_file))
click.echo('Storing data...')
if non_contrib:
evidence_list = _database_select(database_name, EvidenceTable)
database_data = []
for evidence_table in evidence_list:
database_data.append([evidence_table, non_contrib])
_database_compound_add(database_name, TestTable, database_data)
else:
_database_compound_add(database_name, TestTable, database_data)
click.echo('Data successfully uploaded.')
except:
click.echo("Error: Unable to read '%s'." % input_file)
raise
@database.command()
@click.argument('database_name')
@click.pass_context
def clear(ctx, database_name):
'''
Clear jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: This action cannot be undone!\nClear the Job table?"):
try:
_database_remove(database_name, JobTable)
except Exception as e:
click.echo("Error: Unable to clear jobs.")
raise
click.echo('Job table successfully cleared.')
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database_name')
@click.argument('batch_name', type=str)
@click.pass_context
def stage(ctx, database_name, batch_name):
'''
Generate jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: There are still jobs currently pending.\nLoad the Job table?"):
try:
location = database_location(database_name)
create_dir('%s/output/%s' % (location, batch_name))
_database_add(database_name, [BatchTable(batch_name)])
batch = _database_select(database_name,
BatchTable,
BatchTable.name == batch_name)
batch = batch[0] # _database_select returns a list, but this is a single value
tests = _database_select(database_name, TestTable)
_database_add(database_name, [JobTable(t.id_, batch.id_) for t in tests])
except Exception as e:
raise
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database_name')
def status(database_name):
'''
Display information about a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
get_count = lambda Table, expr=True: database_count(database_name, Table, expr)
evidence_count = get_count(EvidenceTable)
comparison_count = get_count(ComparisonTable)
test_count = get_count(TestTable)
job_count = get_count(JobTable)
pending_count = get_count(JobTable, JobTable.event_id == 1)
processing_count = get_count(JobTable, JobTable.event_id == 2)
complete_count = get_count(JobTable, JobTable.event_id == 3)
failed_count = get_count(JobTable, JobTable.event_id == 4)
evidence, comparison = next_job(database_name)
job = '%s and %s' % (evidence.name, comparison.name)
batch = current_batch(database_name).name
echo_title('Database Status', fg='yellow', bold=True)
echo_field('Evidence', evidence_count, fg='yellow')
echo_field('Comparison', comparison_count, fg='yellow')
echo_field('Tests', test_count, fg='yellow')
click.echo('')
echo_field('Jobs', job_count, fg='yellow')
echo_field(' Pending', pending_count, fg='yellow')
echo_field(' Processing', processing_count, fg='yellow')
echo_field(' Complete', complete_count, fg='yellow')
echo_field(' Failed', failed_count, fg='yellow')
click.echo('')
echo_field('Current Batch', batch, fg='yellow')
echo_field('Next Job', job, fg='yellow')
echo_field('Average Time', '23.3 sec', fg='yellow')
# END command-line interface

2
run_cjsd

@ -1,4 +1,4 @@
#!/usr/bin/env bash
cd cjsd
../.venv/bin/gunicorn app:api --reload
../.venv/bin/gunicorn cjsd:api --reload
Loading…
Cancel
Save