Browse Source

init cjsv1

automate2
Stephen Lorenz 4 years ago
parent
commit
62da36e4d1
  1. BIN
      cjs/.data/state.db
  2. 21
      cjs/LICENSE
  3. 1
      cjs/README.md
  4. 0
      cjs/__init__.py
  5. 218
      cjs/automate.py
  6. 51
      cjs/charts.py
  7. 49
      cjs/cjs.py
  8. 0
      cjs/core/__init__.py
  9. 273
      cjs/core/database.py
  10. 66
      cjs/core/reqbt.py
  11. 59
      cjs/core/state.py
  12. 224
      cjs/database.py
  13. 0
      cjs/models/__init__.py
  14. 119
      cjs/models/database.py
  15. 19
      cjs/models/state.py
  16. 44
      cjs/reqbt.py
  17. 6
      cjs/resources/mailer.json
  18. 14
      cjs/resources/settings.json
  19. 77
      cjs/resources/tutorialdata/FreqDatabases/ESX17_Norway.csv
  20. 98
      cjs/resources/tutorialdata/FreqDatabases/Fusion 6C_Norway.csv
  21. 86
      cjs/resources/tutorialdata/FreqDatabases/Identifiler_Caucasian.csv
  22. 51
      cjs/resources/tutorialdata/FreqDatabases/Identifiler_NIST.csv
  23. 86
      cjs/resources/tutorialdata/FreqDatabases/NGM_Holland.csv
  24. 50
      cjs/resources/tutorialdata/FreqDatabases/SGMPlus_Norway.csv
  25. 60
      cjs/resources/tutorialdata/FreqDatabases/SGMPlus_UK.csv
  26. 1310
      cjs/resources/tutorialdata/databaseESX17.txt
  27. BIN
      cjs/resources/tutorialdata/my_project_file.Rdata
  28. 33
      cjs/resources/tutorialdata/refs.csv
  29. 18
      cjs/resources/tutorialdata/stain.txt
  30. 19
      cjs/setup.py
  31. 0
      cjs/utils/__init__.py
  32. 12
      cjs/utils/cli.py
  33. 50
      cjs/utils/fs.py
  34. 82
      cjs/utils/io.py
  35. 23
      cjs/utils/net.py
  36. 13
      euroformix.headless/DESCRIPTION
  37. 2
      euroformix.headless/NAMESPACE
  38. 355
      euroformix.headless/R/efm_headless.R

BIN
cjs/.data/state.db

21
cjs/LICENSE

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2019 Stephen Lorenz
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
cjs/README.md

@ -0,0 +1 @@
# altar

0
cjs/__init__.py

218
cjs/automate.py

@ -0,0 +1,218 @@
#!/usr/bin/env python3
import threading
import click
import subprocess
import shlex
import sys
import queue
import utils.io
from utils.io import read_json, write_csv
import utils.net
from utils.net import send_email
import signal
def signal_handler(sig, frame):
print('Exiting automate.')
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
# from https://www.endpoint.com/blog/2015/01/28/getting-realtime-output-using-python
def run_command(command):
return subprocess.call(command)
# main-entry point to the automate feature
@click.group(name='automate')
@click.option('-w', '--workers',
type=int, default=1,
help='Set number of worker threads.')
@click.pass_context
def automate(ctx, workers):
'''Bulk test systems.'''
ctx.obj['workers'] = workers
def send_error(subject, message):
send_email('resources/mailer.json',
'cucjsoftware@gmail.com',
subject, message)
def _efm(q):
from core.vault import next_job, all_jobs
# load the custom headless euroformix package
'''
rjson = importr('rjson')
euroformix = importr('euroformix.headless')
'''
worker_id = 1
while True:
# get next job
evidence_table, comparison_table = q.get()
# open input files from job
evidence = read_json(evidence_table.path)
comparison = read_json(comparison_table.path)
# convert them to euroformix format
# evidence: reqbt -> euroformix
# initialize csv with header
evidence_header = [
'Sample Name', 'Marker', # NOTE: THIS HAS TWO SPACES - IS THAT OK?
'Allele 1', 'Allele 2', 'Allele 3',
'Allele 4', 'Allele 5', 'Allele 6',
'Height 1', 'Height 2', 'Height 3',
'Height 4', 'Height 5', 'Height 6',
'ADO', 'UD1'
]
comparison_header = [
'SampleName',
'Marker',
'Allele1',
'Allele2'
]
evidence_loci = [
'AMEL',
'D3S1358',
'TH01',
'D21S11',
'D18S51',
'D10S1248',
'D1S1656',
'D2S1338',
'D16S539',
'D22S1045',
'vWA',
'D8S1179',
'FGA',
'D2S441',
'D12S391',
'D19S433',
'SE33'
]
comparison_loci = [
'D3S1358',
'TH01',
'D21S11',
'D18S51',
'D10S1248',
'D1S1656',
'D2S1338',
'D16S539',
'D22S1045',
'VWA', # NOTE: caps?
'D8S1179',
'FGA',
'D2S441',
'D12S391',
'D19S433',
'SE33'
]
from pprint import pprint
for replicate in evidence['replicates']:
tmp_evidence = []
max_values = 6
for i in range(17):
tmp_evidence.append([evidence['name']])
for locus, row in zip(evidence_loci, tmp_evidence):
row.append(locus)
try:
values = replicate[locus]
curr_values = 0
for v in values:
if curr_values > 5: # only allow 6 values
print('Warning: %s may have more than 6 loci values.' % evidence['name'])
break
row.append(v)
curr_values += 1
except KeyError:
row.append('false')
except Exception as e:
print(e)
print('Something went seriously wrong.')
sys.exit(1)
break # NOTE: only get first replicate
pprint(tmp_evidence)
print('----------------------------------------------------------------')
tmp_comparison = []
for i in range(len(comparison_loci)):
tmp_comparison.append([comparison['name']])
for locus, row in zip(comparison_loci, tmp_comparison):
row.append(locus)
try:
if locus == 'VWA':
locus = 'vWA'
values = comparison[locus] # NOTE: use uppercase here
for v in values:
row.append(v)
except KeyError:
row.append('') # append 2 blank values
row.append('')
print('Bad key: %s: Does this exist?' % locus)
pass
except Exception as e:
print(e)
print('Something went seriously wrong.')
sys.exit(1)
pprint(tmp_comparison)
# comparison: reqbt -> euroformix
sys.exit()
try:
# call the entry-point function
# TODO: put settings.json into vault table
#print(euroformix.headless_efm(evidence.path, comparison.path, 'resources/settings.json'))
print('\n' + '-'*64)
err = run_command(['euroformix.headless/R/efm_headless.R', evidence.path, comparison.path])
if err:
send_error('Worker %d: RRuntimeError' % worker_id,
'Something went wrong!\nSorry,\nWorker %d' % (worker_id))
break
print('\n' + '-'*64)
except Exception as e:
send_error('Worker %d: RRuntimeError' % worker_id,
'An exception occurred: %s\nSorry,\nWorker %d' % (e, worker_id))
# main-entry point to the euroformix subcommand
@automate.command()
@click.argument('vault')
@click.pass_context
def efm(ctx, vault):
from core.vault import all_jobs
num_workers = ctx.obj['workers']
q = queue.Queue()
threads = []
for i in range(0, num_workers):
t = threading.Thread(target=_efm, args=(q,))
t.start()
threads.append(t)
jobs = all_jobs(vault)
print('''
\ / _ _ _ _ | _ _
\/\/ | (_|| |(_||(/_|
_| v0.1.0
System: EuroForMix v2.1.0
Total Jobs: %d
Workers: %d
''' % (len(jobs), num_workers), end='')
for job in jobs:
q.put(job)
q.join()
for i in range(num_workers):
q.put(None)
for t in threads:
t.join()

51
cjs/charts.py

@ -0,0 +1,51 @@
#!/usr/bin/env
import matplotlib.pyplot as plt
import csv
import math
from decimal import Decimal
import click
@click.group()
@click.pass_context
def charts(ctx):
'''Create graphs and figures.\f'''
pass
@charts.command()
@click.pass_context
def efm(ctx):
plt.style.use('seaborn-whitegrid')
# x,y axis values
x = []
y = []
# open & read the csv file
with open('resources/graph_data.csv','r') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
#loop through all rows. Puts column (0) in x, & column (1) in y.
for i, row in enumerate(csv_reader):
y.append(math.log10(Decimal(row[0])))
x.append(math.log10(Decimal(row[1])))
plt.rcParams['axes.facecolor'] = '#ededed'
plt.ticklabel_format(style='sci', axis='x')
plt.ticklabel_format(style='sci', axis='y')
# plotting the points
plt.scatter(x,y, s=0.75, color='black')
# naming the x,y axis
plt.xlabel('LR - EuroForMix v2.1.0')
plt.ylabel('LR - Forensic Statistical Tool v2.0')
# giving a title to the graph
plt.title('EuroForMix Vs. Forensic Statistical Tool')
# Graph legend
plt.legend()
# function to show the plot
plt.show()

49
cjs/cjs.py

@ -0,0 +1,49 @@
#!/usr/bin/env python3
import click
# TODO: replace with plugin manager
from database import database
from automate import automate
from reqbt import reqbt
from charts import charts
_commands = [
database,
automate,
reqbt,
charts
]
# main-entry point to the entire command-line interface
@click.group()
@click.option('-v', '--verbose', help='Print additional information.',
is_flag=True)
@click.pass_context
def cli(ctx, verbose):
"""
A command-line interface wrapper to the Criminal Justice Software toolchain.\f
Args:
ctx: Click command-line interface context.
verbose: Flag to enable additional prints.
database: Path to a SQLite3 database.
"""
ctx.ensure_object(dict)
# initialize context object
# store non-click information
# TODO: replace hard coded path with a more cross-platform method
ctx.obj['data_dir'] = '/home/csguest/Desktop/cjsoftware/cjs/.data'
ctx.obj['resource_dir'] = '/home/csguest/Dekstop/cjsoftware/cjs/resources'
# store click-related information
# global flags
ctx.obj['verbose'] = verbose
# load external modules into cli
for cmd in _commands:
cli.add_command(cmd)
if __name__ == '__main__':
cli()

0
cjs/core/__init__.py

273
cjs/core/database.py

@ -0,0 +1,273 @@
#!/usr/bin/env python3
# external
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from sqlalchemy.exc import IntegrityError
import core.state
import utils.fs
from utils.fs import remove_file, abs_path, copy_file, create_dir
import utils.io
from utils.io import write_json, get_valid_filename
import models.state
from models.state import StateVault
import models.database
from models.database import *
# BEGIN vault backend
class Database:
def __init__(self, path):
self._engine = create_engine('sqlite:///%s' % path)
self._SessionFactory = sessionmaker(bind=self._engine)
def _get_session(self):
DatabaseBase.metadata.create_all(self._engine)
return self._SessionFactory
def get_session(func):
def wrapper(self, *args, **kwargs):
s = self._get_session()
res = func(s, *args, **kwargs)
s.close()
return res
return wrapper
def _get(self, table):
s = self.get_session()
res = s.query(table).all()
s.close()
return res
def _clear(self, table):
s = self.get_session()
s.query(table).delete()
s.commit()
s.close()
def get_tests(self):
return self._get(VaultTest)
def clear_jobs(self):
self._clear(VaultJob)
def insert_evidence(self, name, path):
evidence = VaultEvidence(name, path)
self._insert(evidence)
return evidence.id
def insert_comparison(self, name, path):
comparison = VaultComparison(name, path)
self._insert(comparison)
return comparison.id
def insert_test(self, evidence_id, comparison_id):
test = VaultTest(evidence_id, comparison_id)
self._insert(test)
return test.id
def insert_event(self, name):
event = VaultEvent(name)
self._insert(event)
return event.id
def insert_job(self, test_id, batch_id):
job = VaultJob(test_id, batch_id)
self._insert(job)
return job.id
def insert_batch(self, name):
batch = VaultBatch(name)
self._insert(batch)
return batch.id
def insert_result(self, name, path):
result = VaultResult(name, path)
self._insert(result)
return result
# new api
def _insert(self, Table):
s = self.get_session()
try:
res = []
for t in Table:
s.add(t)
s.flush()
s.refresh(t)
res.append(t.id)
except:
s.add(Table)
s.flush()
s.refresh(Table)
res = Table.id
s.commit()
s.close()
return res
def insert(self, Table):
return self._insert(Table)
def _query(self, Table):
s = self.get_session()
q = s.query(Table)
s.close()
return q
def sizeof(self, Table, where=None):
if where:
return self._query(Table).filter(where).count()
else:
return self._query(Table).count()
def next(self, Table):
s = self.get_session()
q = s._query(Table).filter(event_id=1)
def select_all(self, Table):
return self._query(Table).all()
def select_first(self, Table):
return self._query(Table).first()
def select_last(self, Table):
return self._query(Table).order_by(Table.id.desc()).first()
# public functions
def new(database_name, database_root):
try:
path = f'{database_root}/database.db'
s = StateVault(database_name, database_root, path)
core.state.add_state(s)
except:
raise
def list_databases():
try:
databases = core.state.list_state(StateVault)
return databases
except:
raise
def delete(name):
vault = _state_get_by_name(name)
try:
remove_file(vault.root)
except :
pass
_state_remove_by_name(name)
def stage(name, batch):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
batch_id = vault.insert(VaultBatch(batch))
tests = vault.get_tests()
l = [VaultJob(t.id, batch_id) for t in tests]
vault.insert(l)
def upload(name, evidence, comparisons):
vault_state = _state_get_by_name(name)
vault_root = vault_state.root
vault = Vault(vault_state.path)
evidence_name = evidence['name']
evidence_file = '%s/input/%s.json' % (vault_root, evidence_name)
evidence_file = str(abs_path(evidence_file))
write_json(evidence_file, evidence)
evidence_id = vault.insert(VaultEvidence(evidence_name, evidence_file))
for c in comparisons:
comparison_name = c['name']
comparison_file = '%s/input/%s.json' % (vault_root, comparison_name)
comparison_file = str(abs_path(comparison_file))
write_json(comparison_file, c)
comparison_id = vault.insert(VaultComparison(comparison_name, comparison_file))
vault.insert(VaultTest(evidence_id, comparison_id))
def clear(name):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
vault.clear_jobs()
def vault_path(name):
v = _state_get_by_name(name)
return v.path
def vault_name(path):
v = _state_get_by_path(path)
return v.name
def next_job(name):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
s = vault.get_session()
job_row = s.query(VaultJob).filter_by(event_id=1).first()
test_row = s.query(VaultTest).filter_by(id=job_row.test_id).first()
evidence = s.query(VaultEvidence).filter_by(id=test_row.evidence_id).first()
comparison = s.query(VaultComparison).filter_by(id=test_row.comparison_id).first()
s.close()
return [evidence, comparison]
def all_jobs(name):
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
s = vault.get_session()
jobs = []
for job_row in s.query(VaultJob).filter_by(event_id=1).all():
test_row = s.query(VaultTest).filter_by(id=job_row.test_id).first()
evidence = s.query(VaultEvidence).filter_by(id=test_row.evidence_id).first()
comparison = s.query(VaultComparison).filter_by(id=test_row.comparison_id).first()
jobs.append([evidence, comparison])
s.close()
return jobs
def insert_result(vault, name, path):
# get the current batch's id
batch_id = vault.select_last(VaultBatch).id
r = VaultResult(name, path, bath_id)
vault.insert(r)
def copy(name, destination):
vault = _state_get_by_name(name)
copy_file(vault.path, destination)
def status(name):
status = {}
vault_state = _state_get_by_name(name)
vault = Vault(vault_state.path)
s = vault.get_session()
status['num_jobs'] = vault.sizeof(VaultJob)
if status['num_jobs'] == 0:
return status
status['num_pending'] = s.query(VaultJob).filter_by(event_id=1).count()
status['num_processing'] = s.query(VaultJob).filter_by(event_id=2).count()
status['num_complete'] = s.query(VaultJob).filter_by(event_id=3).count()
job_row = s.query(VaultJob).filter_by(event_id=1).first()
test_row = s.query(VaultTest).filter_by(id=job_row.test_id).first()
evidence_name = s.query(VaultEvidence).filter_by(id=test_row.evidence_id).first().name
comparison_name = s.query(VaultComparison).filter_by(id=test_row.comparison_id).first().name
status['next_job'] = [evidence_name, comparison_name]
s.close()
return status

66
cjs/core/reqbt.py

@ -0,0 +1,66 @@
#!/usr/bin/env python3
import marshmallow
from marshmallow import fields
class LociSchema(marshmallow.Schema):
D8S1179 =fields.List(marshmallow.fields.String(), required=True)
D21S11 = fields.List(fields.String(), required=True)
D7S820 = fields.List(fields.String(), required=True)
CSF1PO = fields.List(fields.String(), required=True)
D3S1358 = fields.List(fields.String(), required=True)
TH01 = fields.List(fields.String(), required=True)
D13S317 = fields.List(fields.String(), required=True)
D16S539 = fields.List(fields.String(), required=True)
D2S1338 = fields.List(fields.String(), required=True)
D19S433 = fields.List(fields.String(), required=True)
vWA = fields.List(fields.String(), required=True)
TPOX = fields.List(fields.String(), required=True)
D18S51 = fields.List(fields.String(), required=True)
D5S818 = fields.List(fields.String(), required=True)
FGA = fields.List(fields.String(), required=True)
class ComparisonSchema(LociSchema):
name = fields.String(
required=True
error_messages={'required': "Field 'name' is required."})
class EvidenceSchema(marshmallow.Schema):
class Meta:
strict = True
name = fields.String(
required=True
error_messages={'required': "Field 'name' is required."})
replicates = fields.List(
fields.Nested(LociSchema()),
required=True,
error_messages={'required': "Field 'replicates' is required."})
comparisons = fields.List(
fields.Nested(ComparisonSchema()),
required=True,
error_messages={'required': "Field 'comparisons' is required."})
known_pn = fields.List(
fields.Nested(LociSchema()))
contributors = fields.Integer(
required=True,
error_messages={'required': "Field 'contributors' is required."})
deducible = fields.Boolean(
required=True,
error_messages={'required': "Field 'deducible' is required."})
quantity = fields.Float(
required=True,
error_messages={'required': "Field 'quantity' is required."})
# NOTE: 0 contributors makes no sense
@marshmallow.validates('contributors')
def validate_contributors(self, value):
if value < 1:
raise marshmallow.ValidationError("Field 'contributors' must be greater than or equal to 1.")
# NOTE: a sample of anything less than or equal to 0 makes no sense
@marshmallow.validates('quantity')
def validate_quantity(self, value):
if value <= 0:
raise marshmallow.ValidationError("Field 'quantity' must be greater than 0.")

59
cjs/core/state.py

@ -0,0 +1,59 @@
#!/usr/bin/env python3
# module extended from:
# https://github.com/auth0-blog/sqlalchemy-orm-tutorial/blob/master/examples/common/base.py
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import models.state
from models.state import StateBase
# TODO: replace with project relative path
_engine = create_engine('sqlite:///.data/state.db')
_SessionFactory = sessionmaker(bind=_engine)
# used to connect to the state database
# must close it after calling
def _get_session():
StateBase.metadata.create_all(_engine)
return _SessionFactory()
# decorator to connect to the state database
# will close session after function completes
def pass_session(func):
def wrapper(*args, **kwargs):
s = _get_session()
res = func(s, *args, **kwargs)
s.close()
return res
return wrapper
# using pass_session is more convenient
@pass_session
def add_state(sess, State):
'''Add a new entry to the given Table in the State database.'''
sess.add(State)
sess.commit()
@pass_session
def remove_state(sess, State, filter_expr):
'''Delete an entry that matches the given filter expression
from the given Table in the State database.'''
sess.query(State).filter(filter_expr).delete()
sess.commit()
@pass_session
def select_state(sess, State, filter_expr):
'''Return entries that match the given filter expression
from the given Table in the State database.'''
q = sess.query(State).filter(filter_expr).all() # TODO: make return first, all, and reveresed
return q
@pass_session
def list_state(sess, State):
'''Return all entires in the given Table in the State database.'''
q = sess.query(State).all()
return q

224
cjs/database.py

@ -0,0 +1,224 @@
#!/usr/bin/env python3
# external
import click
import utils.io
from utils.io import read_json, get_valid_filename
import utils.fs
from utils.fs import remove_file, file_exists
import utils.cli
from utils.cli import echo_title, echo_field
import core.database
def init_database_skel(root):
create_dir(root)
create_dir(f'{root}/input')
create_dir(f'{root}/output')
create_dir(f'{root}/log')
# BEGIN command-line interface
@click.group()
@click.pass_context
def database(ctx):
"""
Manage automation databases.\f
Args:
ctx: Click command-line interface context.
"""
# ensure database directory exists
database_dir = '%s/database' % ctx.obj['data_dir']
utils.fs.create_dir(database_dir)
# store database_dir to context object
ctx.obj['database_dir'] = database_dir
@database.command()
@click.argument('database_name')
@click.pass_context
def new(ctx, database_name):
'''
Create a new database.\f
Args:
ctx: Click command-line interface context.
'''
try:
# try to register the new database
core.database.new(database_name)
#new_dir = f'{ctx.obj['database_dir']}/{get_valid_filename(database_name)}'
#init_database_skel(new_dir)
except Exception as e:
# TODO: better exception handling
click.echo("Error: Unable to create '%s': %s" % (db, e))
ctx.exit()
# if database was created successfully, initialize
@database.command(name='list')
@click.pass_context
def list_databases(ctx):
'''
Display available databases.\f
Args:
ctx: Click command-line interface context.
'''
echo_title('Available Databases', fg='yellow', bold=True)
databases = core.database.list_databases()
for i, db in enumerate(databases, start=1):
echo_field(db.name, db.root, fg='yellow', bold=True)
@database.command()
@click.argument('database_name')
@click.pass_context
def delete(ctx, db):
'''
Remove a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
if click.confirm("Warning: This action cannot be undone!\nDelete database? (%s)" % database):
try:
core.database.delete(db)
except:
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
else:
click.echo('No changes were made.')
click.echo('database successfully deleted.')
@database.command()
@click.argument('database')
@click.argument('input_file')
@click.pass_context
def upload(ctx, db, input_file):
'''
Add data to a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
click.echo('Please wait, this may take awhile...')
try:
data = read_json(input_file)
except:
click.echo("Error: Unable to read '%s'." % input_file)
ctx.exit(1)
with click.progressbar(data.items()) as bar:
for slug, evidence in bar:
# break apart data into separate files
comparisons = evidence.pop('comparisons')
core.database.upload(db, evidence, comparisons)
@database.command()
@click.argument('database')
@click.pass_context
def clear(ctx, db):
'''
Clear jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: This action cannot be undone!\nClear the Job table? (%s)" % database):
try:
core.database.clear(db)
except:
click.echo("Error: Unable to clear jobs.")
ctx.exit(1)
click.echo('Job table successfully cleared.')
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database')
@click.argument('batch_name', type=str)
@click.pass_context
def stage(ctx, db, batch_name):
'''
Generate jobs.\f
Args:
ctx: Click command-line interface context.
'''
if click.confirm("Warning: This may cause overlapping jobs!\nLoad the Job table? (%s)" % database):
core.database.stage(db, batch_name)
else:
click.echo('No changes were made.')
@database.command()
@click.argument('database')
def status(db):
'''
Display information about a database.\f
Args:
ctx: Click command-line interface context.
src: Source directory containing input file(s)
'''
from core.database import database_status
s = core.database.status(db)
if s['num_jobs'] > 0:
plabel('Job Tracker', '', 'yellow')
plabel(' Total', s['num_jobs'], 'yellow')
plabel(' Pending', (s['num_pending']/s['num_jobs'])*100, 'yellow')
plabel(' Processing', (s['num_processing']/s['num_jobs'])*100, 'yellow')
plabel(' Complete', (s['num_complete']/s['num_jobs'])*100, 'yellow')
click.echo('')
evidence, comparison = s['next_job']
plabel('Next Job', '%s and %s' % (evidence, comparison), 'yellow')
else:
click.echo('No job currently staged.')
@database.command()
@click.argument('database')
@click.pass_context
def clone(ctx, db):
'''
Copy a database.\f
'''
click.echo('New database:')
name = click.prompt("Name (e.g. 'my_database')", type=str)
while True:
path = click.prompt("Location (e.g. 'path/to/database.db')", type=str)
if not file_exists(path):
break
click.echo("'%s' already exists." % path)
try:
core.database.copy(db, path)
core.database.new(name, path)
except Exception as e:
click.echo(e)
raise
click.echo('Error: Unable to create new database.')
ctx.exit(1) # TODO: change exit code
# END command-line interface

0
cjs/models/__init__.py

119
cjs/models/database.py

@ -0,0 +1,119 @@
#!/usr/bin/env python3
import datetime
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, ForeignKey, DateTime
DatabaseBase = declarative_base()
class VaultEvidence(DatabaseBase):
'''Used to store evidence files.'''
__tablename__ = 'Evidence'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
path = Column('path', String(260), unique=True)
def __init__(self, name, path):
self.name = name
self.path = path
def __str__(self):
return '%s, %s, %s' % (self.id, self.name, self.path)
class VaultComparison(DatabaseBase):
'''Used to store comparison files.'''
__tablename__ = 'Comparison'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
path = Column('path', String(260), unique=True)
def __init__(self, name, path):
self.name = name
self.path = path
def __str__(self):
return '%s, %s, %s' % (self.id, self.name, self.path)
class VaultTest(DatabaseBase):
'''Used to store comparison files.'''
__tablename__ = 'Test'
id = Column('id', Integer, primary_key=True)
evidence_id = Column('evidence_id', ForeignKey('Evidence.id'))
comparison_id = Column('comparison_id', ForeignKey('Comparison.id'))
def __init__(self, name, path):
self.evidence_id = name
self.comparison_id = path
def __str__(self):
return '%s, %s, %s' % (self.id, self.evidence_id, self.comparison_id)
class VaultEvent(DatabaseBase):
'''Used to identify which set that a result belongs to.'''
__tablename__ = 'Event'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
def __init__(self, name):
self.name = name
def __str__(self):
return '%s, %s' % (self.id, self.name)
class VaultJob(DatabaseBase):
'''Used to store comparison files.'''
__tablename__ = 'Job'
id = Column('id', Integer, primary_key=True)
test_id = Column('test_id', ForeignKey('Test.id'))
event_id = Column('event_id', ForeignKey('Event.id'), default=1)
batch_id = Column('batch_id', ForeignKey('Batch.id'))
timestamp = Column('timestamp', DateTime, default=datetime.datetime.utcnow)
def __init__(self, test_id, batch_id):
self.test_id = test_id
self.batch_id = batch_id
class VaultBatch(DatabaseBase):
'''Used to identify which set that a result belongs to.'''
__tablename__ = 'Batch'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
def __init__(self, name):
self.name = name
def __str__(self):
return '%s, %s' % (self.id, self.name)
class VaultResult(DatabaseBase):
'''Used to store output files from completed jobs.'''
__tablename__ = 'Result'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32))
path = Column('path', String(260))
processing_time = Column('processing_time', Integer())
batch_id = Column('batch_id', ForeignKey('Batch.id'))
def __init__(self, name, path, batch_id, time):
self.name = name
self.path = path
self.batch_id = batch_id
self.processing_time = time
def __str__(self):
return '%s, %s, %s' % (self.id, self.name, self.path, self.time)

19
cjs/models/state.py

@ -0,0 +1,19 @@
#!/usr/bin/env python3
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
StateBase = declarative_base()
class StateVault(StateBase):
__tablename__ = 'Vault'
id = Column('id', Integer, primary_key=True)
name = Column('name', String(32), unique=True)
path = Column('path', String(260), unique=True)
root = Column('root', String(260), unique=True)
def __init__(self, name, path, root):
self.name = name
self.root = root
self.path = path

44
cjs/reqbt.py

@ -0,0 +1,44 @@
#!/usr/bin/env python3
# external
import click
from pprint import pprint
import utils.io
@click.group()
@click.pass_context
def reqbt(ctx):
'''Inspect the ReQBT dataset.\f'''
pass
# pretty label
def plabel(label, message, color):
click.echo(click.style(label+':', fg=color, bold=True) + ' %s' % message)
@reqbt.command()
@click.argument('filename')
@click.argument('name')
@click.pass_context
def find(ctx, filename, name):
data = utils.io.read_json(filename)
case = data.get(name, None)
try:
plabel('Name', case['name'], 'yellow')
plabel('Contributors', case['contributors'], 'yellow')
print(click.style('Deducible:', fg='yellow', bold=True),
'Yes' if case['deducible'] else 'No')
plabel('Quantity', case['quantity'], 'yellow')
for i, replicate in enumerate(case['replicates'], start=1):
click.echo(click.style('Replicate %d:', fg='yellow', bold=True) % i)
for key, value in replicate.items():
click.echo(' %-20s %s' % (click.style(key, fg='yellow', bold=True), ', '.join(value)))
for i, comparison in enumerate(case['comparisons'], start=1):
click.echo(click.style('Comparison %d:', fg='yellow', bold=True) % i)
click.echo(' %-20s %s' % (click.style('Name:', fg='yellow', bold=True), comparison.pop('name')))
for key, value in comparison.items():
click.echo(' %-20s %s' % (click.style(key, fg='yellow', bold=True), ', '.join(value)))
except TypeError:
click.echo("Case '%s' not found." % name)

6
cjs/resources/mailer.json

@ -0,0 +1,6 @@
{
"server": "smtp.gmail.com",
"port": 587,
"login": "cucjsoftware@gmail.com",
"password": "%FascinatingFrog6"
}

14
cjs/resources/settings.json

@ -0,0 +1,14 @@
{
"workingDir": "/home/csguest/Desktop/.altar/.data/databases/my_vault",
"databaseFile": "/home/csguest/Desktop/.altar/resources/tutorialdata/FreqDatabases/ESX17_Norway.csv",
"referenceFile": "/home/csguest/Desktop/.altar/resources/tutorialdata/refs.csv",
"nDone": 4,
"conservativeLR": false,
"kit": "ESX17",
"threshT": 200,
"fst": 0.01,
"stutter": true,
"degrad": true,
"dropin": true,
"niter": 2000
}

77
cjs/resources/tutorialdata/FreqDatabases/ESX17_Norway.csv

@ -0,0 +1,77 @@
Allele,D3S1358,TH01,D21S11,D18S51,D10S1248,D1S1656,D2S1338,D16S539,D22S1045,VWA,D8S1179,FGA,D2S441,D12S391,D19S433,SE33
5,,0.00259844093543874,,,,,,,,,,,,,,
6,,0.209274435338797,,,,,,0.000899280575539568,,,,,,,,
6.3,,,,,,,,,,,,,,,,
7,,0.212472516490106,,0.000898472596585804,,,,,,,,,,,,
8,,0.0836498101139316,,,,,,0.00869304556354916,,,0.0113022566077856,,,,,
8.2,,,,,,,,,,,,,,,,0.002506945333238
9,,0.140915450729562,,0.000998302885095338,,,,0.13449240607514,,,0.00920183723819713,,,,,
9.1,,,,,,,,,,,,,,,,
9.3,,0.344293423945633,,,,,,,,,,,,,,
10,0.00089865202196705,0.00589646212272636,,0.0105820105820106,,0.002,,0.052158273381295,,,0.096419251061109,,0.172897,,,
10.1,,,,,,,,,,,,,,,,
11,0.00559161258112831,0.000899460323805717,,0.00638913846461016,0.00467300467300467,0.077,,0.316446842525979,0.154213618152737,,0.0769153569149303,,0.397196,,,
11.1,,,,,,,,,,,,,,,,
11.3,,,,,,,,,,,,,0.037383,,,
11.4,,,,,,,,,,,,,,,,
12,,,,0.132075471698113,0.028037028037028,0.148,,0.273681055155875,0.0186929234304175,0.000898472596585804,0.140027957972565,,0.03271,,0.074766,0.006556626256162
13,0.00329505741387918,,,0.127882599580713,0.285047285047285,0.045,,0.180755395683453,0.00467323085760437,0.000898472596585804,0.337267338773921,,0.018692,,0.242991,0.006556626256162
13.1,,,,,,,,,,,,,0.004673,,,
13.2,,,,,,,,,,,,,,,0.009346,0.002506945333238
13.4,,,,,,,,,,,,,,,,
14,0.124113829256116,,,0.181291803933313,0.303738303738304,0.075,,0.0300759392486011,0.0420076751791538,0.0922431865828092,0.202140359330396,,0.294393,,0.327103,0.0325902893321
14.1,,,,,,,,,,,,,,,,
14.2,,,,,,,,,,,,,,,0.014019,0.002506945333238
15,0.270993509735397,,,0.139862234201857,0.21962621962622,0.133,,0.00279776179056755,0.29440754373266,0.0754716981132076,0.0991197902505799,,0.03271,0.0418247155120783,0.205607,0.035579339537115
15.2,,,,,,,,,,,,,,,0.023364,0.002506945333238
15.3,,,,,,0.07,,,,,,,,,,0.002506945333238
16,0.23155267099351,,,0.116102625536588,0.116822116822117,0.113,0.0373830373830374,,0.397215622451749,0.21842867125886,0.0259051722249245,,0.009346,0.0234577760031943,0.060748,0.047149856459754
16.1,,,,,,,,,,,,,,,,0.002506945333238
16.2,,,,,,,,,,,,,,,0.028037,0.002506945333238
16.3,,,,,,0.065,,,,,,,,0.000898382910760631,,0.002506945333238
17,0.202695956065901,,,0.0971348707197764,0.0420560420560421,0.04,0.233645233645234,,0.0794429244804693,0.294599181391634,0.00170067962559198,0.000896950368746263,,0.129466959472949,0.009346,0.050138906664769
17.2,,,,,,,,,,,,,,,0.004673,
17.3,,,,,,0.153,,,,,,,,0.0143741265721701,,0.002506945333238
18,0.140988517224164,,,0.0849555755216133,,0.002,0.0794390794390794,,0.00934646171520873,0.204252770290506,,0.0248156268686466,,0.177380714713516,,0.062191528459185
18.3,,,,0.000898472596585804,,0.07,,,,,,,,0.010481133958874,,
19,0.017373939091363,,,0.0625935908954777,,,0.107477107477107,,,0.0898472596585804,,0.0603946581622484,,0.114493910960272,,0.077233200458616
19.2,,,,,,,,,,,,0.000896950368746263,,,,0.002506945333238
19.3,,,,,,0.007,,,,,,,,0.00678778199241366,,0.002506945333238
20,0.00159760359460809,,,0.0213636817410402,,,0.163551163551164,,,0.0189677548168114,,0.160354793701415,,0.112297863845079,,0.036639970255024
20.2,,,,,,,,,,,,0.000896950368746263,,,,0.008002940871492
20.3,,,,,,,,,,,,,,0.000898382910760631,,
21,0.00089865202196705,,,0.0070879504841769,,,0.028037028037028,,,0.00259558750124788,,0.169124975084712,,0.0981233779197445,,0.019573457794131
21.1,,,,,,,,,,,,,,,,0.002506945333238
21.2,,,,,,,,,,,,0.00239186764999003,,,,0.02208040312737
22,,,,0.00499151442547669,,,0.028037028037028,,,0.000898472596585804,,0.167630057803468,,0.113795168696347,,0.010991991076507
22.1,,,,,,,,,,,,,,,,0.002506945333238
22.2,,,,,,,,,,,,0.00867052023121387,,,,0.02208040312737
23,,,,0.00139762403913347,,,0.0887850887850888,,,,,0.13832967909109,,0.089239369135556,,0.002506945333238
23.2,,,,,,,,,,,,0.006079330277058,,,,0.033650920050008
24,,,,0.00169711490466207,,,0.0981310981310981,,,0.000898472596585804,,0.133745266095276,,0.0426232780994211,,0.002506945333238
24.2,,,0.000897308075772682,,,,,,,,,0.000996611520829181,,,,0.035097234665338
25,,,0.000897308075772682,0.000898472596585804,,,0.121495121495122,,,,,0.0841140123579829,,0.0154721501297664,,
25.2,,,0.000897308075772682,,,,,,,,,0.000896950368746263,,,,0.042135965793277
26,,,0.00309072781655035,,,,0.014019014019014,,,,,0.0301973290811242,,0.00658814134557796,,0.002506945333238
26.2,,,0.000897308075772682,,,,,,,,,0.000896950368746263,,,,0.046667751587978
27,,,0.0351944167497507,0.000898472596585804,,,,,,,,0.006079330277058,,0.000898382910760631,,
27.2,,,,,,,,,,,,,,,,0.083789826714778
28,,,0.1740777666999,,,,,,,,,0.00169423958540961,,0.000898382910760631,,0.002506945333238
28.2,,,,,,,,,,,,,,,,0.084271931586554
29,,,0.228115653040877,,,,,,,,,0.000896950368746263,,,,
29.2,,,0.000897308075772682,,,,,,,,,,,,,0.079258040920077
29.3,,,0.000897308075772682,,,,,,,,,,,,,
30,,,0.250947158524427,,,,,,,,,,,,,
30.2,,,0.0333000997008973,,,,,,,,,,,,,0.052163747126231
31,,,0.0834496510468594,,,,,,,,,,,,,0.002506945333238
31.2,,,0.0713858424725823,,,,,,,,,,,,,0.028637029383532
32,,,0.0138584247258225,,,,,,,,,,,,,
32.2,,,0.0732801595214357,,,,,,,,,,,,,0.01050988620473
33,,,0.00139581256231306,,,,,,,,,,,,,0.002989050205015
33.2,,,0.0241276171485543,,,,,,,,,,,,,0.002989050205015
34,,,,,,,,,,,,,,,,0.002506945333238
34.2,,,0.00239282153539382,,,,,,,,,,,,,0.002506945333238
35,,,,,,,,,,,,,,,,0.002506945333238
35.2,,,,,,,,,,,,,,,,0.002506945333238
36,,,,,,,,,,,,,,,,0.002506945333238
37,,,,,,,,,,,,,,,,0.002814283217225

98
cjs/resources/tutorialdata/FreqDatabases/Fusion 6C_Norway.csv

@ -0,0 +1,98 @@
Allele,D3S1358,D1S1656,D2S441,D10S1248,D13S317,Penta E,D16S539,D18S51,D2S1338,CSF1PO,Penta D,TH01,vWA,D21S11,D7S820,D5S818,TPOX,D8S1179,D12S391,D19S433,SE33,D22S1045,FGA
2.2,,,,,,,,,,,0.000208834043437481,,,,,,,,,,,,
5,,,,,,0.0736265814902774,,,,,,0.00302524840750924,,,,,0.000104373016177818,,,,,,
6,,,,,,0.000157545960392946,0.00015652691281451,,,,0.00558630116195064,0.214687886988296,,,0.000260879267322985,,0.000104373016177818,,,,,,
6.3,,,,,,,,,,,,,,,0.000156527160393381,,,,,,,,
7,,,,,0.00010434101137317,0.172040956748903,,,,0.000626435679264932,0.00574293119452969,0.208063890475168,,,0.0194615199422195,0.0010433008555067,0.000521866080889243,,,,,,
8,,,0.00115207038594358,,0.118218012885763,0.0127086968050336,0.0101220943619934,,,0.00307996842305617,0.0187428038985032,0.0922695514293081,,,0.159866163814858,0.00114763094105737,0.553908085855753,0.0127282056029561,,,,,
8.1,,,,,,,,,,,,,,,0.000208703213858183,,,,,,,,
8.2,,,,,,,,,,,5.22084108593495e-05,,,,,,,,,,,,
8.3,,,,,,,,,,,,0.000156477917630024,,,5.21761534649044e-05,,,,,,,,
9,,,0.000864055289458522,,0.0754382082227647,0.0139165965013676,0.135917924293716,0.00041744924237103,,0.0221340886673466,0.214890044697129,0.133371929793016,,,0.172180176433027,0.0365676299854566,0.103382016024212,0.00954617420222588,,,,,
9.1,,,0.000288018096486062,,,,,,,,0.000104416021718533,,,,0.000417406427716366,,,,,,,,
9.2,,,,,,,,,,,,,,,,,,,,,0.000158278015194689,,
9.3,,,,,,,,,,,,0.340496820762474,,,,,,,,,,,
9.4,,,,,,,,,,,5.22084108593495e-05,,,,,,,,,,,,
10,5.21594105987922e-05,0.00028801812269572,0.194988065321002,0.00057603712845628,0.0888982096899049,0.105188973555492,0.0544714696593914,0.0124191072105336,,0.247180873443393,0.125614026127717,0.00777171590896875,,,0.251174257378262,0.0587901482079215,0.0545871084610018,0.0913928402311283,,0.000288183973487074,,,
10.1,,,,,,,,,,,,,,,0.000260879267322985,,,,,,,,
10.3,,,,,,,,,,5.22029732720777e-05,,0.000104318945086507,,,5.21761534649044e-05,,,5.21648229629551e-05,,,,,
11,0.0046421909432932,0.0740207315328316,0.384217128712738,0.00432028096342265,0.261582028512441,0.0994116750079049,0.306479829290735,0.00834899484742641,,0.298861846982734,0.123629025714837,5.21593725433063e-05,,,0.209746214926946,0.388106318247181,0.249870038729856,0.0737089324466721,,0.00201728981440934,0.000316556030389379,0.141417056566823,
11.1,,,,,,,,,,,,,,,0.00172180176433027,,,,,,,,
11.2,,,,,,,,,,,,,,,,,,,,,0.000633112060778758,,
11.3,,,0.0403226135080755,,,,,,,,,,,,5.21761534649044e-05,,,,,,,,
11.4,,,,,,,,,,,0.000104416021718533,,,,,,,,,,,,
12,5.21594105987922e-05,0.132200056317224,0.0380184127361683,0.0342742076431483,0.291058031725325,0.157861960313503,0.285609840915319,0.134888078316018,,0.329870831106135,0.234938048867114,,,,0.142022145530093,0.352008288646797,0.0372612057754869,0.1389670611733,,0.069164293636885,0.00791390075973447,0.0112327044930818,
12.1,,,,,,,,,,,,,,,0.000156527160393381,,,,,0.00115273989394793,,,
12.2,,,,,,,,5.21812302964223e-05,,,5.22084108593495e-05,,,,,,,,,0.000288183973487074,0.000633112060778758,,
12.3,,,0.00057603719297246,,,,,,,,,,,,,,,,,,,,
12.4,,,,,,,,,,,5.22084108593495e-05,,,,,,,,,,,,
13,0.00328604066772346,0.0506912215944604,0.0308180103240335,0.315956070458204,0.111644012169197,0.0817664794439071,0.172962903659663,0.126800073620123,,0.0812277584113877,0.194267040407544,,0.000573813211392787,,0.0359491368370805,0.150756123620021,0.000260933040444621,0.325717143380687,,0.225359979266882,0.00902184086609672,0.00518433207373283,
13.1,,,0.000864055289458522,,,,,,,,,,,,,,,,,,,,
13.2,,,,,,,,,,,,,,,,,,,,0.0123918988599453,0.00142450013675201,,
13.3,,,,,,,,,,,0.000104416021718533,,,,,,,,,,,,
13.4,,,,,,,,,,,0.000522084108593495,,,,,,,,,,,,
14,0.132224026867922,0.0783410333732802,0.270449090600445,0.309620069045275,0.0519094056581252,0.0570841856490357,0.0317749823013349,0.18727810873367,,0.0133117931843619,0.0572204119018457,,0.0908712334769624,,0.00584372598806602,0.0100156082127987,,0.215754094974953,0.000104362040200258,0.370604965904343,0.0303894029173827,0.0365783146313259,
14.1,,,,,,,,,,,0.000417668086874962,,,,,,,,,,,,
14.2,,,,,,,,,,,,,,,,,,,,0.0126800988334309,0.000316556030389379,,
14.3,,0.00345622147235035,,,,,,,,,,,,,,,,,,,,,
15,0.26350905354504,0.124136052881959,0.0322581108064671,0.158122035261214,0.00114775012510476,0.0673247830745495,0.00250442860503327,0.142455082709421,0.000288184093948015,0.00360199815577694,0.0146184030406278,,0.083046430594305,,0.000417406427716366,0.00156495128326005,,0.102921045305844,0.044771617246027,0.182132983243766,0.0351377033732195,0.333237133294853,
15.1,,,,,,,,,,,5.22084108593495e-05,,,,,,,,,,,,
15.2,,,,,,,,,,,,,,,,,,,,0.0412103962086436,0.00158278015194689,,
15.3,,0.0846774360725878,,,,,,,,,,,,,,,,,,,0.000316556030389379,,
16,0.236699048097247,0.11117504736057,0.00518433173675113,0.141993031664446,,0.0546684862563426,,0.116260067500595,0.034293911179815,5.22029732720777e-05,0.00167067034749943,,0.206416076043682,,,,,0.026134611504456,0.0279691107737015,0.0507204953337144,0.0440013042241252,0.37413614965446,
16.2,,,,,,,,0.000104362060592612,,,,,,,,,,,,0.0207492980910646,0.00110795010636321,,5.21811703610952e-05
16.3,,0.0521313222079433,,,,,,,,,,,,,,,,,0.000104362040200258,,0.00126622012155713,,
17,0.204048041462562,0.0423387180362939,,0.0311060069366395,,0.0525153867976318,,0.100918058593025,0.20662806736075,,0.00125300026062405,,0.296557109251639,,,,,0.00292123128592601,0.123930047737854,0.00662823939020198,0.0463754044520388,0.0869816347926539,0.00062617364433337
17.1,,0.00028801812269572,,,,,,,,,,,,,,,,,5.2181420100283e-05,,,,
17.2,,,,,,,,,,,,,,,,,,,0.000156544060300772,0.0025936597613833,,,
17.3,,0.152650065028928,,,,,,,,,,,,,,,,,0.0155500059898623,,0.00126622012155713,,
17.4,,,,,,5.25153867976318e-05,,,,,,,,,,,,,,,,,
18,0.13937002831999,0.00316820134965377,,0.00403226089919418,,0.0234743940985373,,0.0769672446871823,0.107781035136617,,0.000104416021718533,,0.210746077638855,,,,,0.000156495068889129,0.174024067034071,0.000288183973487074,0.0685343065792934,0.0100806040322416,0.0232205868107067
18.1,,,,,,,,0.000104362060592612,,,,,,,,,,,,,,,
18.2,,,,,,,,,,,,,,,,,,,,0.00172910984092189,0.000316556030389379,,
18.3,,0.0789171336186989,,,,,,5.21812302964223e-05,,,,,,,,,,,0.0106450041004556,,0.000158278015194689,,
19,0.0150219030524507,,,,,0.00961032758396365,,0.0534335310235081,0.100000032600011,,,,0.0903495332847681,,,,,,0.117929045426268,,0.0694840066704646,0.00115207046082818,0.0602170657967066
19.2,,,,,,,,,,,,,,,,,,,5.2181420100283e-05,,0.00284900027350403,,0.000156543911083059
19.3,,0.0112327047851322,,,,,,,,,,,,,,,,,0.00756630291453988,,,,
20,0.000991029201377134,,,,,0.00824491792722763,,0.0206637119973512,0.151585049416726,,,,0.0192488070912605,,,,,,0.113024043536862,,0.0460589044216548,,0.15513491188337
20.1,,,,,,,,,,,,,,,,,,,,,,,5.21811703610952e-05
20.2,,,,,,,,,,,,,,,,,,,,,0.0131371012611617,,0.000417448762889103
20.3,,0.00028801812269572,,,,,,,,,,,,,,,,,0.00052181420100283,,0.000158278015194689,,
21,0.000104319021197625,,,,,0.0091901976895843,,0.00746191433238746,0.020749306764274,,,,0.00166927061495929,,,,,,0.098362037889057,,0.022950302203229,,0.172510902013808
21.1,,,,,,,,,,,,,,,,,,,,,0.000158278015194689,,
21.2,,,,,,,,,,,,,,,,,,,,,0.0212092020360834,,0.00407012768816747
22,,,,,,0.000945277762357171,,0.00594865345378819,0.0288184093948015,,,,5.21648192175194e-05,,,,,,0.112137043195189,,0.00759734072934471,,0.175380900383649
22.2,,,,,,,,,,,,,,,,,,,,,0.0262741025223138,,0.0082446153170585
23,,,,,,0.000105030973595213,,0.00234815136333668,0.0861671280904838,,,,5.21648192175194e-05,,,,,,0.0881867339695299,,0.00221589021272546,,0.140210920360197
23.2,,,,,,,,,,,,,,,,,,,,,0.0367205035251683,,0.00558338682863628
23.3,,,,,,,,,,,,,,,,,,,,,,,5.21811703610952e-05
24,,,,,,0.000105030973595213,,0.00140889081800201,0.110086035888048,,,,0.000417319153740376,5.2164796922277e-05,,,,,0.0426843164419987,,0.000791390075973447,,0.129617926377018
24.2,,,,,,,,,,,,,,0.00052164796922277,,,,,,,0.0376701036163299,,0.000678354614694579
24.3,,,,,,,,,,,,,,5.2164796922277e-05,,,,,,,,,5.21811703610952e-05
25,,,,,,,,0.00104362060592612,0.119597038988635,,,,,0.0006259779630673,,,,,0.0159674061506449,,,,0.0834375526074701
25.2,,,,,,,,,,,,,,0.00052164796922277,,,,,,,0.0332384031908867,,0.000104361940722418
26,,,,,,,,0.000313087181778418,0.0319885104282544,,,,,0.00245174985534676,,,,,0.00516596198992856,,0.000316556030389379,,0.032195781712796
26.2,,,,,,,,,,,,,,,,,,,,,0.0463754044520388,,5.21811703610952e-05
27,,,,,,,,0.000313087181778418,0.00201729065763675,,,,,0.0393844976763146,,,,,0.000939266361805403,,,,0.00537465694719485
27.2,,,,,,,,,,,,,,,,,,,,,0.0900601086457704,,
28,,,,,,,,,,,,,,0.168283990071245,,,,,0.000156544060300772,,,,0.00219160875516623
28.2,,,,,,,,,,,,,,0.00010432999384453,,,,,,,0.0891105085546088,,
28.3,,,,,,,,,,,,,,5.2164796922277e-05,,,,,,,,,
29,,,,,,,,,,,,,,0.22310898683657,,,,,,,0.000158278015194689,,0.000365267792527894
29.2,,,,,,,,,,,,,,0.00052164796922277,,,,,,,0.0770814073998151,,
29.3,,,,,,,,,,,,,,0.000573812966145035,,,,,,,,,
30,,,,,,,,,,,,,,0.250025985248467,,,,,,,0.000158278015194689,,
30.2,,,,,,,,,,,,,,0.035732897891759,,,,,,,0.0539728051813893,,
31,,,,,,,,,,,,,,0.0794991953095475,,,,,,,0.000474834045584068,,
31.2,,,,,,,,,,,,,,0.0765257954849781,,,,,,,0.0283318027198531,,
32,,,,,,,,,,,,,,0.0152320991013062,,,,,,,0.000791390075973447,,
32.2,,,,,,,,,,,,,,0.075847695524986,,,,,,,0.0132953012763489,,
33,,,,,,,,,,,,,,0.00156494990766796,,,,,,,0.00284900027350403,,
33.2,,,,,,,,,,,,,,0.0263431984457513,,,,,,,0.00395695037986724,,
34,,,,,,,,,,,,,,0.00010432999384453,,,,,,,0.00221589021272546,,
34.1,,,,,,,,,,,,,,5.2164796922277e-05,,,,,,,,,
34.2,,,,,,,,,,,,,,0.00266040984303582,,,,,,,0.000949668091168137,,
35,,,,,,,,,,,,,,0.00010432999384453,,,,,,,0.00300728028869891,,
35.2,,,,,,,,,,,,,,5.2164796922277e-05,,,,,,,0.000474834045584068,,
37,,,,,,,,,,,,,,,,,,,,,0.000316556030389379,,
47,,,,,,,,,,,,,,,,,,,,,0.0147198014131009,,

86
cjs/resources/tutorialdata/FreqDatabases/Identifiler_Caucasian.csv

@ -0,0 +1,86 @@
Allele,D8S1179,D21S11,D7S820,CSF1PO,D3S1358,TH01,D13S317,D16S539,D2S1338,D19S433,vWA,TPOX,D18S51,D5S818,FGA
1,,0.000121242,,,0.000121242,,0.000424345,0.000242483,0.000121242,,0.000242483,,6.06E-05,,0.000181862
4,,,,,,,,,,,,,,,
5,,,,,,0.002546072,,,,,,0.000181862,,,
6,,,,,,0.223508729,,,,,,0.000424345,,,
6.3,,,0.000363725,,,,,,,,,,,,
7,,,0.018004365,0.000363725,,0.185378274,0.000242483,6.06E-05,,,,0.000242483,,0.000484966,
7.3,,,,0.000121242,,,,,,,,,,,
8,0.018913676,,0.15306741,0.002788555,,0.104207081,0.119907856,0.016367604,,,,0.525278855,6.06E-05,0.002242968,
8.1,,,6.06E-05,,,,,,,,,,,,
8.3,,,,,,0.000606208,,,,,,,,,
9,0.0128516,,0.163797284,0.023460233,,0.140033948,0.075412221,0.114997575,,,,0.112269641,0.000606208,0.036433075,
9.1,,,6.06E-05,,,,,,,,,,,,
9.3,,,6.06E-05,,,0.334869059,,,,,,,,,
10,0.101418526,,0.263821532,0.270186712,0.000242483,0.008729389,0.065652279,0.057771581,,0.000484966,,0.056195441,0.011336081,0.059590204,
10.1,,,6.06E-05,,,,,,,,,,,,
10.3,,,,0.000363725,6.06E-05,,,,,,,,,,
11,0.076867119,,0.206656159,0.297951018,0.001515519,,0.299042192,0.307165373,,0.003091659,,0.264609602,0.013154704,0.373120757,
11.1,,,,,,,,,,,,,,,
11.2,,,,,,,,,,6.06E-05,,,,,
11.3,,,0.000121242,,,,,,,,,,,,
12,0.139185257,,0.151370029,0.318986421,0.000484966,,0.284675073,0.297829777,,0.079595053,0.000242483,0.04007032,0.141306984,0.351297284,
12.1,,,,,,,,,,0.000545587,,,,,
12.2,,,,,,,,,,0.000969932,,,,,
13,0.324987876,,0.035038797,0.068016489,0.003091659,,0.107419981,0.178225024,0.000424345,0.238906402,0.000666828,0.000606208,0.117179922,0.163069835,
13.1,,,,,,,,,,,,,6.06E-05,,
13.2,,,,,,,,,,0.011760427,,,,,
13.3,,,,,,0.000121242,,,,,,,,,
14,0.193016489,,0.006971387,0.015094568,0.129061591,,0.045586809,0.026066925,0.000363725,0.355480116,0.110147915,0.000121242,0.163069835,0.011578565,
14.2,,,,,,,,,,0.022369059,,,6.06E-05,,
15,0.097902522,,0.000484966,0.002606693,0.271096023,,0.00163676,0.001212415,0.000303104,0.170526188,0.110451018,,0.145186712,0.002121726,0.000181862
15.2,,,,,,,,,,0.039706596,,,,,
16,0.031159069,,6.06E-05,6.06E-05,0.254485936,,,6.06E-05,0.038676043,0.048617847,0.210050921,,0.139791465,,
16.2,,,,,,,,,,0.018368089,,,6.06E-05,,
17,0.003455383,,,,0.194350145,,,,0.20192774,0.00478904,0.267216295,,0.11942289,6.06E-05,0.000606208
17.1,,,,,6.06E-05,,,,,,,,,,
17.2,,,,,,,,,,0.003879728,,,,,
18,0.000242483,,,,0.133729389,,,,0.084202231,0.000303104,0.201260912,,0.076745878,,0.016731329
18.1,,,,,,,,,,,,,0.000181862,,
18.2,,,,,,,,,,0.000424345,,,6.06E-05,,
18.3,,,,,,,,,,,6.06E-05,,,,
19,,,,,0.011032978,,,,0.11614937,6.06E-05,0.081898642,,0.04249515,,0.062984966
19.2,,,,,,,,,,6.06E-05,,,,,0.000121242
19.3,,,,,,,,,,,,,,,6.06E-05
20,,,,,0.000606208,,,,0.142276916,,0.0160645,,0.017216295,,0.139912706
20.1,,,,,6.06E-05,,,,,,,,,,
20.2,,,,,,,,,,,,,,,0.002061106
21,,,,,,,,,0.034614452,,0.001697381,,0.007092629,,0.176406402
21.2,,,,,,,,,,,,,,,0.00242483
22,,,,,,,,,0.032735209,,,,0.002970417,,0.175800194
22.1,,,,,,,,,,,,,,,6.06E-05
22.2,,,,,,,,,,,,,,,0.00885063
22.3,,6.06E-05,,,,,,,,,,,,,
23,,,,,,,,,0.100509214,,,,0.001091174,,0.150824442
23.2,,,,,,,,,,,,,,,0.003758487
24,,,,,,,,,0.11614937,,,,0.000484966,,0.132638215
24.2,,0.001091174,,,,,,,,,,,,,0.001091174
24.3,,,,,,,,,,,,,,,0.000121242
25,,0.000181862,,,,,,,0.108268671,,,,0.000181862,,0.083838506
25.2,,0.001091174,,,,,,,,,,,,,0.000363725
26,,0.001758002,,,,,,,0.021096023,,,,0.000121242,,0.032432105
26.2,,0.000121242,,,,,,,,,,,,,
27,,0.035038797,,,,,,,0.001879243,,,,,,0.007092629
28,,0.160402522,,,,,,,0.000303104,,,,,,0.001273036
28.2,,,,,,,,,,,,,,,
29,,0.210050921,,,,,,,,,,,,,0.000181862
29.1,,0.000242483,,,,,,,,,,,,,
29.2,,0.000606208,,,,,,,,,,,,,
29.3,,0.00078807,,,,,,,,,,,,,
30,,0.259153734,,,,,,,,,,,,,
30.2,,0.032432105,,,,,,,,,,,,,
30.3,,,,,,,,,,,,,,,
31,,0.072381183,,,,,,,,,,,,,
31.2,,0.089839961,,,,,,,,,,,,,
32,,0.014306499,,,,,,,,,,,,,
32.1,,6.06E-05,,,,,,,,,,,,,
32.2,,0.085960233,,,,,,,,,,,,,
33,,0.001091174,,,,,,,,,,,,,
33.2,,0.029340446,,,,,,,,,,,,,
34,,6.06E-05,,,,,,,,,,,,,
34.1,,0.000242483,,,,,,,,,,,,,
34.2,,0.002485451,,,,,,,,,,,,,
35,,0.000242483,,,,,,,,,,,,,
35.2,,0.000606208,,,,,,,,,,,,,
36.1,,6.06E-05,,,,,,,,,,,,,
36.2,,0.000181862,,,,,,,,,,,,,

51
cjs/resources/tutorialdata/FreqDatabases/Identifiler_NIST.csv

@ -0,0 +1,51 @@
Allele,D8S1179,D21S11,D7S820,CSF1PO,D3S1358,TH01,D13S317,D16S539,D2S1338,D19S433,vWA,TPOX,D18S51,D5S818,FGA
5,,,,,,0.00166,,,,,,0.00166,,,
6,,,,,,0.23179,,,,,,0.00166,,,
7,,,0.01821,,,0.1904,,,,,,,,0.00166,
8,0.01159,,0.15066,0.00497,,0.08444,0.11258,0.01821,,,,0.53477,,0.00331,
8.1,,,0.00166,,,,,,,,,,,,
9,0.00331,,0.17715,0.01159,,0.11424,0.0745,0.11258,,,,0.11921,,0.04967,
9.3,,,,,,0.36755,,,,,,,,,
10,0.10099,,0.24338,0.21689,,0.00828,0.05132,0.05629,,0.00166,,0.05629,0.00828,0.05132,
11,0.08278,,0.20695,0.30132,0.00166,0.00166,0.3394,0.32119,,0.00497,,0.24338,0.01656,0.36093,
12,0.18543,,0.16556,0.36093,,,0.24834,0.32616,,0.08113,,0.04139,0.12748,0.38411,
12.2,,,,,,,,,,0.00166,,,,,
13,0.30464,,0.03477,0.09603,,,0.12417,0.1457,,0.25331,0.00166,0.00166,0.13245,0.14073,
13.2,,,,,,,,,,0.00662,,,,,
14,0.16556,,0.00166,0.00828,0.10265,,0.04801,0.01987,,0.36921,0.09437,,0.13742,0.00662,
14.2,,,,,,,,,,0.01821,,,0.00166,,
15,0.11424,,,,0.26159,,0.00166,,0.00166,0.15232,0.11093,,0.15894,0.00166,
15.2,,,,,,,,,,0.03477,,,,,
16,0.03146,,,,0.25331,,,,0.03311,0.04967,0.20033,,0.13907,,
16.2,,,,,,,,,,0.0149,,,,,
17,,,,,0.21523,,,,0.18212,0.00828,0.28146,,0.12583,,
17.2,,,,,,,,,,0.00166,,,,,
18,,,,,0.15232,,,,0.07947,,0.20033,,0.07616,,0.02649
18.2,,,,,,,,,,0.00166,,,,,
19,,,,,0.01159,,,,0.11424,,0.1043,,0.03808,,0.05298
20,,,,,0.00166,,,,0.1457,,0.00497,,0.02152,,0.12748
21,,,,,,,,,0.04139,,0.00166,,0.00828,,0.18543
21.2,,,,,,,,,,,,,,,0.00497
22,,,,,,,,,0.03808,,,,0.00828,,0.21854
22.2,,,,,,,,,,,,,,,0.01159
23,,,,,,,,,0.11755,,,,,,0.13411
23.2,,,,,,,,,,,,,,,0.00331
24,,,,,,,,,0.12252,,,,,,0.13576
24.2,,,,,,,,,,,,,,,0.00166
25,,,,,,,,,0.09272,,,,,,0.07119
25.2,,0.00166,,,,,,,,,,,,,
26,,,,,,,,,0.0298,,,,,,0.02318
27,,0.02649,,,,,,,0.00166,,,,,,0.00331
28,,0.15894,,,,,,,,,,,,,
29,,0.19536,,,,,,,,,,,,,
29.2,,0.00331,,,,,,,,,,,,,
30,,0.27815,,,,,,,,,,,,,
30.2,,0.02815,,,,,,,,,,,,,
31,,0.08278,,,,,,,,,,,,,
31.2,,0.09934,,,,,,,,,,,,,
32,,0.00662,,,,,,,,,,,,,
32.2,,0.08444,,,,,,,,,,,,,
33,,0.00166,,,,,,,,,,,,,
33.2,,0.02649,,,,,,,,,,,,,
34.2,,0.00497,,,,,,,,,,,,,
35,,0.00166,,,,,,,,,,,,,

86
cjs/resources/tutorialdata/FreqDatabases/NGM_Holland.csv

@ -0,0 +1,86 @@
Allele;D10S1248;VWA;D16S539;D2S1338;D8S1179;D21S11;D18S51;D22S1045;D19S433;TH01;FGA;D2S441;D3S1358;D1S1656;D12S391
2.2;;;;;;;;;;;;;;;
5;;;;;;;;;;0.007194245;;;;;
6;;;;;;;;;;0.209832134;;;;;
7;0.000239808;;0.000239808;;;;;;;0.180815348;;;;;
8;;;0.016786571;;0.019184652;;;;;0.111031175;;0.000479616;;;
8.3;;;;;;;;;;0.000239808;;;;;
9;;;0.129496403;;0.016786571;;0.000959233;;;0.135251799;;0.000239808;;;
9.3;;;;;;;;;;0.346522782;;;;;
10;0.000719424;0.000239808;0.065227818;;0.088968825;;0.007913669;0.001438849;0.000479616;0.00911271;;0.188729017;0.000239808;0.001678657;
10.2;;;;;;;;;;;;;;;
10.3;;;;;;;;;;;;;;;
11;0.00263789;;0.321582734;;0.077458034;;0.013908873;0.140527578;0.005035971;;;0.343165468;0.001918465;0.074820144;
11.2;;;;;;;;;;;;;;;
11.3;;;;;;;;;;;;0.042446043;;;
12;0.035491607;0.000479616;0.264268585;0.000239808;0.159472422;;0.157074341;0.013429257;0.066906475;;;0.047482014;;0.123980815;
12.1;;;;;0.000239808;;;;0.000959233;;;;;;
12.2;;;;;;;;;0.000239808;;;;;;
12.3;;;;;;;;;;;;0.001678657;;;
13;0.316067146;0.001918465;0.179376499;;0.319664269;;0.137889688;0.005995204;0.213429257;;;0.025659472;0.005995204;0.057314149;
13.2;;;;;;;;;0.01822542;;;;;;
13.3;;;;;;;;;;;;0.000479616;;;
13.4;;;;;;;;;;;;;;;
14;0.307194245;0.098800959;0.022541966;0.000239808;0.194964029;;0.164748201;0.050839329;0.36618705;;;0.293764988;0.122781775;0.087529976;0.000239808
14.1;;;;;;;;;;;;0.000239808;;;
14.2;;;;;;;;;0.023261391;;;;;;
14.3;;;;;;;;;;;;;;0.001678657;
15;0.196402878;0.097601918;0.000239808;0.000479616;0.091846523;;0.142925659;0.325179856;0.182973621;;;0.049640288;0.246522782;0.13117506;0.041486811
15.2;;;;;;;;;0.042206235;;;;;;
15.3;;;;;;;;;;;;;;0.074820144;
16;0.117266187;0.205995204;0.000239808;0.038129496;0.026378897;;0.123261391;0.370743405;0.05323741;;;0.005995204;0.241726619;0.118705036;0.037410072
16.1;;;;;;;;;;;;;;0.000239808;
16.2;;;;;;;;;0.016786571;;;;;;
16.3;;;;;;;;;;;;;;0.054916067;0.000239808
17;0.020383693;0.274100719;;0.194004796;0.005035971;;0.107434053;0.084172662;0.005275779;;0.001199041;;0.216306954;0.057793765;0.100479616
17.1;;;;;;;;;;;;;;0.000719424;
17.2;;;;;;;;;0.003357314;;;;;;
17.3;;;;;;;;;;;;;;0.141966427;0.022781775
18;0.003357314;0.199760192;;0.083693046;;;0.071702638;0.006714628;0.000239808;;0.017985612;;0.154436451;0.00383693;0.175539568
18.2;;;;;;;;;0.000959233;;;;;;
18.3;;;;;;;;;;;;;;0.053717026;0.02206235
19;0.000239808;0.105995204;;0.116546763;;;0.036690647;0.000959233;;;0.060671463;;0.009352518;0.000479616;0.103117506
19.1;;;;;;;;;;;0.000479616;;;;
19.2;;;;;;;;;0.000239808;;0.000959233;;;;
19.3;;;;;;;;;;;;;;0.013669065;0.009832134
20;;0.014388489;;0.151558753;;;0.019664269;;;;0.134052758;;0.000719424;;0.120623501
20.1;;;;;;;;;;;;;;;
20.2;;;;;;;;;;;0.000239808;;;;0.000239808
20.3;;;;;;;;;;;;;;0.000959233;0.000479616
21;;0.000719424;;0.030455635;;;0.00911271;;;;0.166906475;;;;0.130935252
21.2;;;;;;;;;;;0.003117506;;;;
22;;;;0.031414868;;;0.004316547;;;;0.172182254;;;;0.101438849
22.1;;;;;;;;;;;0.000239808;;;;
22.2;;;;;;;;;;;0.007434053;;;;
23;;;;0.093764988;;;0.000959233;;;;0.147242206;;;;0.075059952
23.2;;;;;;;;;;;0.003357314;;;;
24;;;;0.122781775;;;0.001438849;;;;0.13764988;;;;0.036930456
24.2;;;;;;0.000239808;;;;;0.001199041;;;;
25;;;;0.112470024;;;;;;;0.10263789;;;;0.01558753
25.2;;;;;;0.000239808;;;;;;;;;
25.3;;;;;;;;;;;;;;;
26;;;;0.021582734;;0.001678657;;;;;0.037410072;;;;0.003597122
26.2;;;;;;;;;;;;;;;
27;;;;0.001678657;;0.042925659;;;;;0.004796163;;;;0.001438849
27.2;;;;;;;;;;;;;;;
27.3;;;;;;;;;;;;;;;
28;;;;0.000959233;;0.169304556;;;;;0.000239808;;;;0.000239808
28.2;;;;;;0.000239808;;;;;;;;;
29;;;;;;0.201199041;;;;;;;;;0.000239808
29.2;;;;;;0.000719424;;;;;;;;;
29.3;;;;;;0.000479616;;;;;;;;;
30;;;;;;0.258752998;;;;;;;;;
30.2;;;;;;0.032853717;;;;;;;;;
31;;;;;;0.091606715;;;;;;;;;
31.2;;;;;;0.07793765;;;;;;;;;
32;;;;;;0.016067146;;;;;;;;;
32.2;;;;;;0.073381295;;;;;;;;;
33;;;;;;0.002877698;;;;;;;;;
33.2;;;;;;0.026139089;;;;;;;;;
34;;;;;;0.000239808;;;;;;;;;
34.1;;;;;;0.000479616;;;;;;;;;
34.2;;;;;;0.001918465;;;;;;;;;
35;;;;;;0.000479616;;;;;;;;;
35.2;;;;;;0.000239808;;;;;;;;;
36;;;;;;;;;;;;;;;
37;;;;;;;;;;;;;;;

50
cjs/resources/tutorialdata/FreqDatabases/SGMPlus_Norway.csv