Browse Source

nearly refactored

cjs3
Stephen Lorenz 3 years ago
parent
commit
65e0430a8a
  1. 3
      cjs/cjs/__init__.py
  2. 14
      cjs/cjs/app.py
  3. 96
      cjs/cjs/automate.py
  4. 2
      cjs/cjs/core/batch.py
  5. 36
      cjs/cjs/core/interface.py
  6. 53
      cjs/cjs/main.py
  7. 2
      cjs/setup.py
  8. 2
      cjsd/setup.py
  9. 31
      common/http.py
  10. 0
      utils/LICENSE
  11. 0
      utils/README.md
  12. 2
      utils/setup.py
  13. 0
      utils/utils/cli.py
  14. 0
      utils/utils/fs.py
  15. 0
      utils/utils/io.py
  16. 2
      utils/utils/net.py
  17. 0
      utils/utils/sys.py

3
cjs/cjs/__init__.py

@ -0,0 +1,3 @@
import pathlib
PKG_DIR = pathlib.Path(__file__).parent

14
cjs/cjs/app.py

@ -1,14 +0,0 @@
#!/usr/bin/env python3
import pathlib
class AppState:
def __init__(self, address, port, working_dir):
self.address = address
self.port = port
self.working_dir = working_dir

96
cjs/cjs/automate.py

@ -1,48 +1,64 @@
#!/usr/bin/env python3
# standard library
# from standard library
import time
import signal
import pathlib
import subprocess
# pip
# from python package index
import click
# local
import cjs.core.automate
from cjs.core.automate import generic_worker
from cjs.core.automate import spawn_workers
from cjs.core.automate import run_process
from cjs.core.automate import send_error
# from local modules/packages
import utils.io
from utils.io import write_csv
from utils.io import read_json
from utils.io import write_json
import cjs.fmt.efm
from cjs.fmt.efm import reqbt_to_evidence
from cjs.fmt.efm import reqbt_to_comparison
from cjs.fmt.efm import reqbt_to_settings
# TODO: implement a better way of handling PKG_dir
# NOTE: see __init__ for PKG_DIR
from . import PKG_DIR
from .core.batch import generic_worker
from .core.batch import spawn_workers
from .core.batch import run_process
from .core.batch import send_error
from .core.interface import AutomateInterface
from .fmt.efm import reqbt_to_evidence
from .fmt.efm import reqbt_to_comparison
from .fmt.efm import reqbt_to_settings
# BEGIN: Command-line interface wrapper
# Main-entry point to the automate interface
# main-entry point to the automate interface
@click.group(name='automate')
@click.option('--num_workers',
type=click.INT,
default=1,
help='Set the number of worker threads.')
@click.option('--resource_dir',
type=click.STRING,
default='.app',
help='Set the working directory.')
@click.option(
'--num_workers',
type=click.IntRange(1), # 1 to inf (not literal inf)
default=1,
show_default=True,
help='Set the amount of worker threads.')
@click.option(
'--resource_dir',
type=click.Path(
file_okay=False,
resolve_path=True
),
default=PKG_DIR/'.resources', # NOTE: see __init__ for PKG_DIR
show_default=True,
help='Set the working directory.')
@click.pass_context
def cli(ctx, num_workers, data_dir, resource_dir):
def cli(ctx, num_workers, resource_dir):
'''Bulk test systems.'''
ctx.obj['num_workers'] = num_workers
ctx.obj['data_dir'] = data_dir
ctx.obj['resource_dir'] = resource_dir
# add additional arguments and options to click's context object
# NOTE: this may be inefficient, but increases readability
ctx.obj = {
**ctx.obj, # include previous key: value pairs
'num_workers': num_workers,
'resource_dir': resource_dir
}
# Main-entry point to the EuroForMix subcommand
@cli.command()
@ -50,21 +66,23 @@ def cli(ctx, num_workers, data_dir, resource_dir):
@click.pass_context
def efm(ctx, database_name):
'''Run multisampleUsage.R.\f'''
# unpack the click context object
# alias the context dictionary to a new variable
address = ctx.obj['address']
port = ctx.obj['port']
database = ctx.obj['database']
interface = AutomateInterface(address, port)
num_workers = ctx.obj['num_workers']
data_dir = ctx.obj['data_dir']
resource_dir = ctx.obj['resource_dir']
database = {
'host': ctx.obj['host'],
'name': database_name
}
try:
# spawn a size num_workers worker cluster of efm workers
spawn_workers(efm_qualitative,
database,
data_dir,
resource_dir,
num_workers)
spawn_workers(
efm_qualitative,
database,
data_dir,
resource_dir,
num_workers)
except Exception as e:
raise

2
cjs/cjs/core/automate.py → cjs/cjs/core/batch.py

@ -27,8 +27,6 @@ from utils.fs import create_dir
import utils.net
from utils.net import send_email
import web
class FileSystemScanner(PatternMatchingEventHandler):
patterns = ["*.csv"]
ignore_patterns = []

36
cjs/cjs/core/web.py → cjs/cjs/core/interface.py

@ -1,36 +1,48 @@
#!/usr/bin/env python3
import json
import utils.http
from utils.http import rest_post
from utils.http import rest_get
import requests
class WebInterface:
def __init__(self, address, port):
def __init__(self, address, port, database):
# TODO: add hostname and port validation
# TODO: find a better way to format them
self.address = 'http://%s:%s' % (address, port)
self.HEADERS = {
'Content-Type': 'application/json'
}
# a decorator that concatenates the host and route for simplicity
def resolve_route(func):
def wrapper(self, route, *args, **kwargs):
# TODO: find a better way to format them
route = self.host + route
return func(self, route, *args, **kwargs)
return wrapper
@resolve_route
def post(self, route, payload):
# TODO: find a better way to format them
response = rest_post(route, payload)
try:
response = requests.post(
route,
headers=self.HEADERS,
json=payload
)
except Exception as e:
raise
content = response.content.decode('utf-8')
data = json.loads(content)
return data
@resolve_route
def get(self, route):
response = rest_get(route)
try:
response = requests.get(
route,
headers=self.HEADERS
)
except Exception as e:
raise
content = response.content.decode('utf-8')
data = json.loads(content)
return data
@ -45,4 +57,8 @@ class AutomateInterface(WebInterface):
def next_job(self, database):
payload = {'database': database}
data = self.post('/automate/next_job', payload)
return data
return data
class DatabaseInterface(WebInterface):
pass

53
cjs/cjs/main.py

@ -1,44 +1,67 @@
#!/usr/bin/env python3
import os
# python package index
import click
# TODO: replace with plugin manager if possible
from . import database
from . import automate
# local modules
COMMAND_LIST = [
database.cli,
automate.cli
]
# TODO: implement a plugin manager for simplicity's sake
#from . import database
from . import automate
# main-entry point to the entire command-line interface
@click.group()
@click.option(
'--address',
default='127.0.0.1',
help='Set the address of cjs daemon'
show_default=True,
type=click.STRING, # TODO: implement type.IPaddress
help='Set the address of cjs daemon.'
)
@click.option(
'--port',
type=click.IntRange(1, 65535),
default='8000',
help='Set the port of the cjs daemon'
show_default=True,
help='Set the port of the cjs daemon.'
)
@click.option(
'--working_dir',
type=click.Path(
file_okay=False,
resolve_path=True
),
default='./cjs-data',
show_default=True,
help='Set the working directory of cjs.'
)
@click.pass_context
def cli(ctx, address, port):
def cli(ctx, address, port, working_dir):
"""
A command-line interface to Clarkson University's Criminal Justice Software toolchain.\f
Args:
ctx: :mod:`click` application context.
"""
# enforce that the context object is a dictionary
# enforce that click's context object is a dictionary
ctx.ensure_object(dict)
# store necessary arguments and options into the dictionary
# initialize a dictionary with cli's arguments and options
# for click's context object
ctx.obj = {
'address': address,
'port': port,
'working_dir': working_dir
}
COMMAND_LIST = [
# database.cli,
automate.cli
]
# register each module in the COMMAND_LIST to cli's click group
# register each module's click group in the COMMAND_LIST
# to cli's click group
for command in COMMAND_LIST:
cli.add_command(command)

2
cjs/setup.py

@ -24,7 +24,7 @@ setup(
],
entry_points={
'console_scripts': [
'cjs = cjs.app:cli',
'cjs = cjs.main:cli',
]
},
classifiers=[

2
cjsd/setup.py

@ -19,7 +19,7 @@ setup(
],
entry_points={
'console_scripts': [
'cjs = cjs.app:cli',
'cjs = cjs.main:cli',
]
}
)

31
common/http.py

@ -1,31 +0,0 @@
#!/usr/bin/env python3
import json
import requests
HEADERS = {
'Content-Type': 'application/json'
}
def rest_post(address, payload, **kwargs):
try:
response = requests.post(
address,
headers=HEADERS,
json=payload,
**kwargs
)
return response
except Exception as e:
raise
def rest_get(address, **kwargs):
try:
response = requests.get(
address,
headers=HEADERS,
**kwargs
)
return response
except Exception as e:
raise

0
common/LICENSE → utils/LICENSE

0
common/README.md → utils/README.md

2
common/setup.py → utils/setup.py

@ -6,7 +6,7 @@ with open('README.md', 'r') as fh:
long_description = fh.read()
setuptools.setup(
name='common',
name='utils',
version='1.0.0',
author='Stephen Lorenz',
author_email='lorenzsj@clarkson.edu',

0
common/cli.py → utils/utils/cli.py

0
common/fs.py → utils/utils/fs.py

0
common/io.py → utils/utils/io.py

2
common/net.py → utils/utils/net.py

@ -3,7 +3,7 @@
import smtplib
import ssl
from utils.io import read_json
from .io import read_json
def send_email(account, receiver, subject, message):
a = read_json(account)

0
common/sys.py → utils/utils/sys.py

Loading…
Cancel
Save