Browse Source

init

master
Stephen Lorenz 3 years ago
commit
9542589e89
  1. 110
      .gitignore
  2. 21
      cjsd/LICENSE
  3. 1
      cjsd/README.md
  4. 3
      cjsd/cjsd/__init__.py
  5. 0
      cjsd/cjsd/controllers/__init__.py
  6. 16
      cjsd/cjsd/controllers/database.py
  7. 0
      cjsd/cjsd/core/__init__.py
  8. 169
      cjsd/cjsd/core/database.py
  9. 65
      cjsd/cjsd/core/state.py
  10. 51
      cjsd/cjsd/main.py
  11. 0
      cjsd/cjsd/models/__init__.py
  12. 13
      cjsd/cjsd/models/database.py
  13. 36
      cjsd/cjsd/models/state.py
  14. 0
      cjsd/cjsd/resources/__init__.py
  15. 62
      cjsd/cjsd/resources/database.py
  16. 0
      cjsd/docs/.keep
  17. 31
      cjsd/setup.py
  18. 0
      cjsd/tests/.keep
  19. 5
      run_cjsd
  20. 23
      setup

110
.gitignore

@ -0,0 +1,110 @@
# cjs
cjs-data/
# cjsd
cjsd/cjsd/.app
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/

21
cjsd/LICENSE

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Stephen Lorenz
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
cjsd/README.md

@ -0,0 +1 @@
# altar

3
cjsd/cjsd/__init__.py

@ -0,0 +1,3 @@
import pathlib
PKG_DIR = pathlib.Path(__file__).parent

0
cjsd/cjsd/controllers/__init__.py

16
cjsd/cjsd/controllers/database.py

@ -0,0 +1,16 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
# ...
# from local packages
# ...
# from local modules
# ...
def new_controller(database):
print(database)

0
cjsd/cjsd/core/__init__.py

169
cjsd/cjsd/core/database.py

@ -0,0 +1,169 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
# ...
# from local packages
import models.state
from models.state import DatabaseModel
# from local modules
from .state import pass_session
from .state import add_state
# BEGIN: Generic Database API
def register_database(name, path):
pass
def deregister_database(name, path):
pass
'''
# modified from https://stackoverflow.com/a/36944992
# NOTE: ideally this would be inside the database class
def _pass_session(fn):
@wraps(fn) # get around the fact a decorator cannot have self
def wrapper(self, *fn_args, **fn_kwargs):
s = self._get_session()
res = fn(self, s, *fn_args, **fn_kwargs)
s.close()
return res
return wrapper
class Database:
def __init__(self, location):
self.location = location # working directory of the database
self._engine = create_engine('sqlite:///%s/database.db' % location)
self._SessionFactory = sessionmaker(bind=self._engine)
def _get_session(self):
DatabaseBase.metadata.create_all(self._engine)
return self._SessionFactory()
# TODO: replace this function with a better method of doing this
@_pass_session
def initialize(self, sess):
# NOTE: this method was created because I was unable to get sqlalchemy's
# initial values to work
names = [
'Pending',
'Processing',
'Complete',
'Failed'
]
events = [EventTable(n) for n in names]
self.add_tables(events)
@_pass_session
def add_tables(self, sess, Tables):
try:
for T in Tables:
sess.add(T)
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def bulk_compound_tables(self, sess, Table, relations):
# NOTE: this may not be the best way to do it, but it is a much faster
# way of inserting the evidence + comparison tables into test
# compared to cjs version 1 where it would do it 1 by 1.
# it accepts this input: [PrimaryTable, [SubTable1, SubTable2]]
# and inserts the them as such in Table(PrimaryTable.id_, SubTable.id_)
try:
for r in relations:
left_table, Ts = r
sess.add(left_table)
for right_table in Ts:
sess.add(right_table)
sess.flush()
sess.add(Table(left_table.id_, right_table.id_))
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def add_compound_tables(self, sess, Table, relation):
# NOTE: this may not be the best way to do it, but it is a much faster
# way of inserting the evidence + comparison tables into test
# compared to cjs version 1 where it would do it 1 by 1.
# it accepts this input: [PrimaryTable, [SubTable1, SubTable2]]
# and inserts the them as such in Table(PrimaryTable.id_, SubTable.id_)
try:
left_table, Ts = relation
sess.add(left_table)
for right_table in Ts:
sess.add(right_table)
sess.flush()
sess.add(Table(left_table.id_, right_table.id_))
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def remove_table(self, sess, Table, expr=True):
try:
sess.query(Table).filter(expr).delete()
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def update_tables(self, sess, Table, expr, values):
try:
sess.query(Table).filter(expr).update(values)
sess.commit()
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def select_tables(self, sess, Table, expr=True):
try:
q = sess.query(Table).filter(expr).all()
return q
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def first_record(self, sess, Table, expr=True):
'''Get first record in the given Table.'''
try:
q = sess.query(Table).filter(expr).first()
return q
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def last_record(self, sess, Table, expr=True):
'''Get last record in the given Table.'''
try:
q = sess.query(Table).filter(expr).order_by(Table.id_.desc()).first()
return q
except:
# TODO: implement fine-grained handling
raise
@_pass_session
def count_table(self, sess, Table, expr=True):
try:
q = sess.query(Table).filter(expr).count()
return q
except:
# TODO: implement fine-grained handling
raise
# END: Generic Database API
'''

65
cjsd/cjsd/core/state.py

@ -0,0 +1,65 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
# from local packages
import models.state
from models.state import StateBase
# from local modules
# ...
# this module is extended from:
# https://github.com/auth0-blog/sqlalchemy-orm-tutorial/blob/master/examples/common/base.py
# TODO: replace with project relative path
_engine = create_engine('sqlite:///.app/state.db')
_SessionFactory = sessionmaker(bind=_engine)
# used to connect to the state database
# must close it after calling
def get_session():
StateBase.metadata.create_all(_engine)
return _SessionFactory()
# decorator to connect to the state database
# will close session after function completes
def pass_session(fn):
def wrapper(*args, **kwargs):
s = _get_session()
res = fn(s, *args, **kwargs)
s.close()
return res
return wrapper
# using pass_session is more convenient
@_pass_session
def add_state(sess, state_table):
'''Add a new entry to the given Table in the State database.'''
sess.add(state_table)
sess.commit()
@_pass_session
def remove_state(sess, state_table, expr=True):
'''Delete an entry that matches the given filter expression
from the given Table in the State database.'''
sess.query(state_table).filter(expr).delete()
sess.commit()
@_pass_session
def update_state(sess, state_table, expr):
pass
@_pass_session
def select_state(sess, state_table, expr=True):
'''Return entries that match the given filter expression
from the given Table in the State database.'''
q = sess.query(state_table).filter(expr).all() # TODO: make return first, all, and reveresed
return q

51
cjsd/cjsd/main.py

@ -0,0 +1,51 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
import falcon
from falcon import API
# from local packages
import resources
from resources import database
# from local modules
import cjsd
from cjsd import PKG_DIR
# TODO: serve overall status of the cjsd
class AppResource:
def on_get(self, req, resp):
resp.media = {
'version': '3.0.0',
'pkg_dir': str(PKG_DIR)
}
ROUTES = {
'/': AppResource,
'/database/new': database.NewResource,
}
def setup_api():
'''Main entry-point to cjsd.'''
# initialize falcon's API framework
api = API()
# uri: A Uniform Resource Identifier is a string of characters that
# unambiguously identifies a particular resource.
# e.g. /my/resource
#
# res: Any resource object
# e.g. AppResource
for uri, res in ROUTES.items():
api.add_route(uri, res())
return api
# either uWSGI or gunicorn will attempt to load this variable
api = application = setup_api()

0
cjsd/cjsd/models/__init__.py

13
cjsd/cjsd/models/database.py

@ -0,0 +1,13 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
# ...
# from local packages
# ...
# from local modules
# ...

36
cjsd/cjsd/models/state.py

@ -0,0 +1,36 @@
#!/usr/bin/env python3
import sqlalchemy.ext.declarative
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy
from sqlalchemy import Column
from sqlalchemy import Integer
from sqlalchemy import String
StateBase = declarative_base()
class DatabaseModel(StateBase):
__tablename__ = 'Database'
id_ = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
path = Column('path', String(260), unique=True)
def __init__(self, name, path):
self.name = name
self.path = path
def __repr__(self):
return 'DatabaseModel({}, {}, {})'.format(
self.id_,
self.name,
self.path,
)
def __str__(self):
return '{}, {}, {}'.format(
self.id_,
self.name,
self.path,
)

0
cjsd/cjsd/resources/__init__.py

62
cjsd/cjsd/resources/database.py

@ -0,0 +1,62 @@
#!/usr/bin/env python3
# from standard library
# ...
# from python package index
import falcon
from falcon import API
import sqlalchemy.exc
from sqlalchemy.exc import IntegrityError
# from local packages
import controllers.database
from controllers.database import new_controller
# from local modules
# ...
# BEGIN: Falcon hooks
def ensure_param(req, resp, Resource, params, key):
'''Try to store value from media into context given a key.'''
try:
req.context[key] = req.media[key]
except TypeError: # No paramaters were provided
raise falcon.HTTPMissingParam(key)
except KeyError: # Missing required parameter
raise falcon.HTTPMissingParam(key)
# END: Falcon hooks
# BEGIN: Database resources
class NewResource:
@falcon.before(ensure_param, 'database')
def on_post(self, req, resp):
db = req.context['database']
try:
new_controller(db)
## determine the new database's location
#database_dir = get_valid_filename(database_name)
#database_path = '.app/database/%s' % database_dir # TODO: un-hardcore path
## initialize the directory tree in the given path
#database_skeleton(database_path)
## register the new database to the state database
#add_database(database_name, database_path)
resp.media = {
'title': 'New database',
'description': 'The creation of "%s" was successful.' % db
}
except IntegrityError:
raise falcon.HTTPBadRequest(
'Database already exists',
'Unable to create database. Try deleting the database first.'
)
except Exception as e:
raise falcon.HTTPInternalServerError()
# END: Database resources

0
cjsd/docs/.keep

31
cjsd/setup.py

@ -0,0 +1,31 @@
#!/usr/bin/env python3
import setuptools
from setuptools import setup
from setuptools import find_packages
with open('README.md', 'r') as fh:
long_description = fh.read()
setup(
name='cjsd',
version='1.1.0',
author='Stephen Lorenz',
author_email='lorenzsj@clarkson.edu',
description='A backend server to the cjs client.',
long_description=long_description,
long_description_content_type='text/markdown',
url='https://github.com/lorenzsj/cjs',
packages=find_packages(),
install_requires=[
'falcon',
'sqlalchemy',
'marshmallow',
'gunicorn'
],
classifiers=[
'Programming Language :: Python :: 3',
'License :: OSI Approved :: MIT License',
'Operating System :: OS Independent'
]
)

0
cjsd/tests/.keep

5
run_cjsd

@ -0,0 +1,5 @@
#!/usr/bin/env bash
cd cjsd/cjsd
../.env/bin/gunicorn main:api --reload

23
setup

@ -0,0 +1,23 @@
#!/usr/bin/env bash
# where the source for cjs and cjsd are located
CJS_DIR="./cjs"
CJSD_DIR="./cjsd"
# where the virtual environment will be located
CJS_ENV="$CJS_DIR/.env"
CJSD_ENV="$CJSD_DIR/.env"
# create a virtual environment and activate it
#virtualenv -p python3 "$CJS_ENV"
#source "$CJS_ENV/bin/activate"
## install the necessary software
#pip install -e "$CJS_DIR"
## deactivate the virtual environment
#deactivate
virtualenv -p python3 "$CJSD_ENV"
source "$CJSD_ENV/bin/activate"
pip install -e "$CJSD_DIR"
# deactivate the virtual environment
deactivate
Loading…
Cancel
Save