Browse Source

added basic resources

cjs3
Stephen Lorenz 3 years ago
parent
commit
8c80d0b06e
  1. BIN
      cjsd/.app/database/q/database.db
  2. BIN
      cjsd/.app/database/tester/database.db
  3. BIN
      cjsd/.app/state.db
  4. 10
      cjsd/app.py
  5. 8
      cjsd/client.py
  6. 238
      cjsd/database.py

BIN
cjsd/.app/database/q/database.db

BIN
cjsd/.app/database/tester/database.db

BIN
cjsd/.app/state.db

10
cjsd/app.py

@ -6,7 +6,7 @@ import json
import falcon
from falcon import API
from database import NewDatabaseResource
import database
PKG_DIR = os.path.dirname(os.path.abspath(__file__))
@ -21,7 +21,13 @@ class AppResource:
ROUTES = {
'/': AppResource,
'/database/new': NewDatabaseResource,
'/database/new': database.NewResource,
'/database/delete': database.DeleteResource,
'/database/list': database.ListResource,
'/database/clone': database.CloneResource,
'/database/clear': database.ClearResource,
'/database/upload': database.UploadResource,
'/database/status': database.StatusResource
}
def create_api():

8
cjsd/client.py

@ -0,0 +1,8 @@
#!/usr/bin/env python3
import json
import requests
headers = {
'Content-Type': 'application/json'
}

238
cjsd/database.py

@ -4,6 +4,9 @@ import json
# pip modules
import click
import falcon
from sqlalchemy.exc import IntegrityError
# local modules
import utils.cli
@ -62,37 +65,207 @@ def database_skeleton(root):
print('Error: Unable to initialize database skeleton: %s' % e)
raise
# BEGIN command-line interface
'''
class DatabaseManager:
_CHUNK_SIZE_BYTES = 1024
# BEGIN: REST API Resources
def get_database_context(req, resp, Resc, params):
try:
req.context['database'] = req.media['database']
# TODO: add database does not exist check
except KeyError:
raise falcon.HTTPMissingParam('database')
class NewResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
# determine the new database's location
database_dir = get_valid_filename(database_name)
database_path = '.app/database/%s' % database_dir # TODO: un-hardcore path
# initialize the directory tree in the given path
database_skeleton(database_path)
# register the new database to the state database
add_database(database_name, database_path)
except IntegrityError:
raise falcon.HTTPBadRequest('Database already exists',
'Unable to create database. Try deleting the database first.')
except Exception as e:
raise falcon.HTTPInternalServerError()
def __init__(self):
self.database = None
class DeleteResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
# determine the existing database's location
database_path = database_location(database_name)
def get_database(fn):
def wrapper(*args, **kwargs):
db =
# delete/clean up it's data directory
remove_dir(database_path)
def connect(self, database_name):
pass
# unregister database from the state database
remove_database(database_name)
except IntegrityError:
raise falcon.HTTPBadRequest('Database does not exist',
'Unable to delete database. Try creating the database first.')
except Exception as e:
raise falcon.HTTPInternalServerError()
class ListResource:
def on_get(self, req, resp):
try:
# get list of all available database objects from state database
database_list = list_database()
# extract relevant information to send to client
resp.media = [db.name for db in database_list]
except IntegrityError:
raise falcon.HTTPBadRequest('No databases to list',
'Unable to list databases. Try creating a database first.')
except Exception as e:
raise falcon.HTTPInternalServerError()
class CloneResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
# get the clone database's name from request
clone_name = req.media['clone']
# determine the new database's location
clone_dir = get_valid_filename(clone_name)
clone_path = '.app/database/%s' % clone_dir
# determine the existing database's location
database_path = database_location(database_name)
# copy contents recursively to the clone's directory
copy_dir(database_path, clone_path)
# register the clone database to the state database
add_database(clone_name, clone_path, init=False)
except IndexError:
raise falcon.HTTPBadRequest('Database does not exist',
'Unable to create database. Try creating the database first.')
except KeyError:
raise falcon.HTTPMissingParam('clone')
except IntegrityError:
raise falcon.HTTPBadRequest('Database already exists',
'Unable to create database. Try deleting the database first.')
except Exception as e:
raise falcon.HTTPInternalServerError()
def disconnect(self):
pass
'''
class ClearResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
_database_remove(database_name, JobTable)
except IntegrityError:
raise falcon.HTTPBadRequest('Table empty',
'Unable to clear Job table. Try staging the table first.')
except Exception as e:
raise falcon.HTTPInternalServerError()
class NewDatabaseResource:
class UploadResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
database_name = req.media.get('database_name')
# determine the existing database's location
database_path = database_location(database_name)
input_path = lambda root, filename: '%s/input/%s.json' % (root, filename)
# read the input data into a dictionary
input_data = read_json(input_file)
database_data = []
non_contrib = []
for key, value in input_data.items():
tmp_data = []
# TODO: implement models.reqbt handler for validation
try:
# create a new copy of the current data
data = dict(value)
# try to see if it's a known contributor
# separate the comparison(s)
comparisons = data.pop('comparisons')
evidence = data # rest of the data
# TODO: clean this up
evidence_name = evidence['name']
evidence_file = input_path(location, evidence_name)
write_json(evidence_file, evidence) # save file
tmp_comparison = []
for comparison in comparisons:
comparison_name = comparison['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison) # save file
tmp_comparison.append(ComparisonTable(comparison_name, comparison_file))
database_data.append([EvidenceTable(evidence_name, evidence_file), tmp_comparison])
# try to see if it's a non-contributors
except ValueError:
data = list(value)
for comparison_dict in data:
comparison_name = comparison_dict['name']
comparison_file = input_path(location, comparison_name)
write_json(comparison_file, comparison_dict) # save file
non_contrib.append(ComparisonTable(comparison_name, comparison_file))
click.echo('Storing data...')
if non_contrib:
evidence_list = _database_select(database_name, EvidenceTable)
database_data = []
for evidence_table in evidence_list:
database_data.append([evidence_table, non_contrib])
_database_compound_add(database_name, TestTable, database_data)
else:
_database_compound_add(database_name, TestTable, database_data)
click.echo('Data successfully uploaded.')
except:
click.echo("Error: Unable to read '%s'." % input_file)
raise
resp.body = json.dumps({
'database': database_name
})
class StatusResource:
@falcon.before(get_database_context)
def on_post(self, req, resp):
database_name = req.context['database']
try:
get_count = lambda Table, expr=True: database_count(database_name, Table, expr)
evidence_count = get_count(EvidenceTable)
comparison_count = get_count(ComparisonTable)
test_count = get_count(TestTable)
job_count = get_count(JobTable)
if job_count:
pending_count = get_count(JobTable, JobTable.event_id == 1)
processing_count = get_count(JobTable, JobTable.event_id == 2)
complete_count = get_count(JobTable, JobTable.event_id == 3)
failed_count = get_count(JobTable, JobTable.event_id == 4)
evidence, comparison = next_job(database_name)
job = '%s and %s' % (evidence.name, comparison.name)
batch = current_batch(database_name).name
except AttributeError:
raise falcon.HTTPBadRequest('Database empty',
'Unable to provide status. Try uploading data first.')
except IntegrityError:
raise falcon.HTTPBadRequest('Table empty',
'Unable to clear Job table. Try staging the table first.')
except Exception as e:
raise
raise falcon.HTTPInternalServerError()
@click.group()
@click.pass_context
@ -104,7 +277,7 @@ def database(ctx):
ctx: Click command-line interface context.
"""
database_dir = '%s/database' % ctx.obj['data_dir']
# store database_dir to context object
ctx.obj['database_dir'] = database_dir
@ -126,19 +299,7 @@ def new(ctx, database_name):
database_name: Name used to register a new database.
'''
try:
# determine where the database will be stored
database_root = get_valid_filename(database_name)
new_location = '%s/%s' % (ctx.obj['database_dir'],
database_root)
database_skeleton(new_location)
# register the new database to the created directory
add_database(database_name, new_location)
except Exception as e:
# TODO: better exception handling
click.echo("Error: Unable to create new database: %s" % e)
raise
@database.command()
@click.argument('database_name')
@ -381,11 +542,4 @@ def status(database_name):
echo_field('Next Job', job, fg='yellow')
echo_field('Average Time', '23.3 sec', fg='yellow')
# END command-line interface
@database.command()
@click.argument('database_name')
def debug(database_name):
from core.database import set_job_status
set_job_status(database_name, 1, 3)
# END command-line interface
Loading…
Cancel
Save