Browse Source

moving computers

cjs3
Stephen Lorenz 3 years ago
parent
commit
927b2f6131
  1. 3
      cjs/cjs/core/batch.py
  2. 72
      cjs/cjs/core/interface.py
  3. 110
      cjs/cjs/database.py
  4. 0
      cjs/docs/.keep
  5. 0
      cjs/tests/.keep
  6. 15
      cjsd/cjsd/resources/database.py
  7. 0
      cjsd/docs/.keep
  8. 0
      cjsd/tests/.keep

3
cjs/cjs/core/batch.py

@ -109,3 +109,6 @@ def spawn_workers(target_func, State, num_workers, *args, **kwargs):
thread_list.append(t) # store thread object
return thread_list
def spawn_foreman(State):
pass

72
cjs/cjs/core/interface.py

@ -1,8 +1,10 @@
#!/usr/bin/env python3
# from standard library
import json
import requests
class WebInterface:
def __init__(self, address, port):
# TODO: add hostname and port validation
@ -16,7 +18,7 @@ class WebInterface:
def resolve_route(func):
def wrapper(self, route, *args, **kwargs):
# TODO: find a better way to format them
route = self.host + route
route = self.address + route
return func(self, route, *args, **kwargs)
return wrapper
@ -31,8 +33,10 @@ class WebInterface:
except Exception as e:
raise
content = response.content.decode('utf-8')
data = json.loads(content)
return data
print(content)
if content:
data = json.loads(content)
return data
@resolve_route
def get(self, route):
@ -44,8 +48,9 @@ class WebInterface:
except Exception as e:
raise
content = response.content.decode('utf-8')
data = json.loads(content)
return data
if content:
data = json.loads(content)
return data
class AutomateInterface(WebInterface):
@ -61,4 +66,59 @@ class AutomateInterface(WebInterface):
class DatabaseInterface(WebInterface):
pass
def new(self, database):
payload = {
'database': database
}
data = self.post('/database/new', payload)
return data
def clone(self, database, duplicate):
payload = {
'database': database,
'clone': duplicate
}
data = self.post('/database/clone', payload)
return data
def delete(self, database):
payload = {
'database': database
}
data = self.post('/database/delete', payload)
return data
def list_available(self):
data = self.get('/database/list')
return data
def upload(self, database, type_, data):
payload = {
'database': database,
'type': type_,
'data': data
}
data = self.post('/database/upload', payload)
return data
def clear(self, database):
payload = {
'database': database
}
data = self.post('/database/clear', payload)
return data
def stage(self, database, batch):
payload = {
'database': database,
'batch': batch
}
data = self.post('/database/stage', payload)
return data
def status(self, database):
payload = {
'database': database,
}
data = self.post('/database/status', payload)
return data

110
cjs/cjs/database.py

@ -1,13 +1,19 @@
#!/usr/bin/env python3
# from standard library
import json
import requests
# from python package index
import click
# from local modules/packages
from .core.interface import DatabaseInterface
import utils.io
from utils.io import read_json
@click.group(name='database')
@click.pass_context
def cli(ctx):
@ -17,7 +23,16 @@ def cli(ctx):
Args:
ctx: Click command-line interface context.
"""
pass
address = ctx.obj['address']
port = ctx.obj['port']
ctx.obj = {
**ctx.obj,
'interface': DatabaseInterface(
address,
port
)
}
@cli.command()
@click.argument('database_name')
@ -30,15 +45,8 @@ def new(ctx, database_name):
ctx: Click command-line interface context.
database_name: Name used to register a new database.
'''
# construct dictionary containing data to be sent to the host
payload = {
'database': database_name
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/new' % ctx.obj['host']
response = rest_post(address, payload)
interface = ctx.obj['interface']
interface.new(database_name)
@cli.command()
@click.argument('database_name')
@ -53,14 +61,8 @@ def clone(ctx, database_name, new_database):
database_name: Name of an existing database.
new_database: Name used to register a new database.
'''
payload = {
'database': database_name,
'clone': new_database
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/clone' % ctx.obj['host']
response = rest_post(address, payload)
interface = ctx.obj['interface']
interface.clone(database_name, new_database)
@cli.command()
@click.argument('database_name')
@ -73,14 +75,8 @@ def delete(ctx, database_name):
ctx: Click command-line interface context.
database_name: Name of an pre-existing database.
'''
# construct dictionary containing data to be sent to the host
payload = {
'database': database_name
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/delete' % ctx.obj['host']
response = rest_post(address, payload)
interface = ctx.obj['interface']
interface.delete(database_name)
@cli.command(name='list')
@click.pass_context
@ -91,10 +87,13 @@ def my_list(ctx):
Args:
ctx: Click command-line interface context.
'''
address = '%s/database/list' % ctx.obj['host']
response = rest_get(address)
print(response.content.decode('utf-8'))
interface = ctx.obj['interface']
database_list = interface.list_available()
if database_list:
for database in database_list:
print(database)
else:
print('No available databases.')
@cli.command()
@click.argument('database_name')
@ -109,24 +108,24 @@ def upload(ctx, database_name, input_file):
database_name: Name of an pre-existing database.
input_file: Path to a CJS format file.
'''
interface = ctx.obj['interface']
# read json data into a dictionary
input_data = read_json(input_file)
# initialize variables needed to send a post request
address = '%s/database/upload' % ctx.obj['host']
payload = {
'database': database_name
}
# print an overall progress bar for each dataset
with click.progressbar(input_data.items()) as bar:
for key, value in bar:
# update data handling method to the payload
payload['type'] = value['type']
type_ = value['type']
with click.progressbar(value['data']) as bar:
for item in bar:
payload['data'] = item
rest_post(address, payload)
for data in bar:
interface.upload(
database_name,
type_,
data
)
@cli.command()
@click.argument('database_name')
@ -139,13 +138,8 @@ def clear(ctx, database_name):
ctx: Click command-line interface context.
database_name: Name of an pre-existing database.
'''
payload = {
'database': database_name,
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/clear' % ctx.obj['host']
response = rest_post(address, payload)
interface = ctx.obj['interface']
interface.clear(database_name)
@cli.command()
@click.argument('database_name')
@ -160,14 +154,8 @@ def stage(ctx, database_name, batch_name):
database_name: Name of an pre-existing database.
batch_name: Name used to identify a staged batch.
'''
payload = {
'database': database_name,
'batch': batch_name
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/stage' % ctx.obj['host']
response = rest_post(address, payload)
interface = ctx.obj['interface']
interface.stage(database_name, batch_name)
@cli.command()
@click.argument('database_name')
@ -180,11 +168,5 @@ def status(ctx, database_name):
ctx: Click command-line interface context.
database_name: Name of an pre-existing database.
'''
payload = {
'database': database_name,
}
# send (POST) the dictonary to the host and store the response
address = '%s/database/status' % ctx.obj['host']
response = rest_post(address, payload)
print(response.content.decode('utf-8'))
interface = ctx.obj['interface']
print(interface.status(database_name))

0
cjs/docs/.keep

0
cjs/tests/.keep

15
cjsd/cjsd/resources/database.py

@ -300,18 +300,21 @@ class StatusResource:
resp.media['complete_count'] = get_count(JobTable, JobTable.event_id == 3)
resp.media['failed_count'] = get_count(JobTable, JobTable.event_id == 4)
evidence, comparison = next_job(database_name)
job, evidence, comparison = next_job(database_name)
resp.media['job'] = '%s and %s' % (evidence.name, comparison.name)
resp.media['batch'] = current_batch(database_name).name
except AttributeError as e:
raise falcon.HTTPBadRequest('Database empty',
'Unable to provide status. Try uploading data first.')
raise falcon.HTTPBadRequest(
'Database empty',
'Unable to provide status. Try uploading data first.'
)
except IntegrityError:
raise falcon.HTTPBadRequest('Table empty',
'Unable to clear Job table. Try staging the table first.')
raise falcon.HTTPBadRequest(
'Table empty',
'Unable to clear Job table. Try staging the table first.'
)
except Exception as e:
raise
raise falcon.HTTPInternalServerError()
class StageResource:

0
cjsd/docs/.keep

0
cjsd/tests/.keep

Loading…
Cancel
Save