Browse Source

fixed job queue issue with being slow. Still has speed issue, but fills while threads work

master
Stephen Lorenz 3 years ago
parent
commit
f88e755a82
  1. BIN
      cjs/.app/database/bug/database.db
  2. 2
      cjs/.app/database/bug/output/test/qualitative_reqbt-1-1.csv
  3. 2
      cjs/.app/database/bug/output/test/qualitative_reqbt-1-2.csv
  4. 2
      cjs/.app/database/bug/output/test/qualitative_reqbt-2-2.csv
  5. 1
      cjs/.app/database/bug/tmp/reqbt-1-1-settings.json
  6. 16
      cjs/.app/database/bug/tmp/reqbt-1-1.csv
  7. 1
      cjs/.app/database/bug/tmp/reqbt-1-2-settings.json
  8. 16
      cjs/.app/database/bug/tmp/reqbt-1-2.csv
  9. 42
      cjs/.app/database/bug/tmp/reqbt-1.tsv
  10. 1
      cjs/.app/database/bug/tmp/reqbt-2-1-settings.json
  11. 16
      cjs/.app/database/bug/tmp/reqbt-2-1.csv
  12. 1
      cjs/.app/database/bug/tmp/reqbt-2-2-settings.json
  13. 16
      cjs/.app/database/bug/tmp/reqbt-2-2.csv
  14. 29
      cjs/.app/database/bug/tmp/reqbt-2.tsv
  15. 1
      cjs/.app/database/bug/tmp/reqbt-3-1-settings.json
  16. 16
      cjs/.app/database/bug/tmp/reqbt-3-1.csv
  17. 31
      cjs/.app/database/bug/tmp/reqbt-3.tsv
  18. 8
      cjs/automate.py
  19. 8
      cjs/core/automate.py
  20. 39
      cjs/core/database.py

BIN
cjs/.app/database/bug/database.db

2
cjs/.app/database/bug/output/test/qualitative_reqbt-1-1.csv

@ -0,0 +1,2 @@
"EvidFile" "POI" "log10LR"
"reqbt-1" "reqbt-1-1" "9.72328273603544e-20"

2
cjs/.app/database/bug/output/test/qualitative_reqbt-1-2.csv

@ -0,0 +1,2 @@
"EvidFile" "POI" "log10LR"
"reqbt-1" "reqbt-1-2" "77410906044.1592"

2
cjs/.app/database/bug/output/test/qualitative_reqbt-2-2.csv

@ -0,0 +1,2 @@
"EvidFile" "POI" "log10LR"
"reqbt-2" "reqbt-2-2" "711627443.60097"

1
cjs/.app/database/bug/tmp/reqbt-1-1-settings.json

@ -0,0 +1 @@
{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1-1.csv", "test_name": "reqbt-1-1", "evidence_name": "reqbt-1", "comparison_name": "reqbt-1-1", "num_contributors": 2, "num_replicates": 3}

16
cjs/.app/database/bug/tmp/reqbt-1-1.csv

@ -0,0 +1,16 @@
SampleName,Marker,Allele1,Allele2
reqbt-1-1,D13S317,11,11
reqbt-1-1,D18S51,15,24
reqbt-1-1,D19S433,13,15
reqbt-1-1,TH01,6,9
reqbt-1-1,FGA,18,21
reqbt-1-1,vWA,17,19
reqbt-1-1,D3S1358,16,17
reqbt-1-1,D16S539,11,11
reqbt-1-1,D2S1338,20,20
reqbt-1-1,D5S818,9,11
reqbt-1-1,D7S820,10,12
reqbt-1-1,TPOX,9,11
reqbt-1-1,D21S11,29,29
reqbt-1-1,D8S1179,14,15
reqbt-1-1,CSF1PO,10,10

1
cjs/.app/database/bug/tmp/reqbt-1-2-settings.json

@ -0,0 +1 @@
{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1-2.csv", "test_name": "reqbt-1-2", "evidence_name": "reqbt-1", "comparison_name": "reqbt-1-2", "num_contributors": 2, "num_replicates": 3}

16
cjs/.app/database/bug/tmp/reqbt-1-2.csv

@ -0,0 +1,16 @@
SampleName,Marker,Allele1,Allele2
reqbt-1-2,D13S317,12,12
reqbt-1-2,D18S51,15,22
reqbt-1-2,D19S433,12,16
reqbt-1-2,TH01,6,9
reqbt-1-2,FGA,24,26
reqbt-1-2,vWA,14,18
reqbt-1-2,D3S1358,15,18
reqbt-1-2,D16S539,9,11
reqbt-1-2,D2S1338,18,19
reqbt-1-2,D5S818,11,12
reqbt-1-2,D7S820,11,11
reqbt-1-2,TPOX,11,11
reqbt-1-2,D21S11,29,31
reqbt-1-2,D8S1179,10,12
reqbt-1-2,CSF1PO,10,12

42
cjs/.app/database/bug/tmp/reqbt-1.tsv

@ -0,0 +1,42 @@
Sample Name Marker Allele 1 Allele 2 Allele 3 Allele 4 ADO UD1
reqbt-1_1 D13S317 12 false
reqbt-1_1 D18S51 15 17 22 false
reqbt-1_1 D19S433 12 13 14 16 false
reqbt-1_1 TH01 6 9 false
reqbt-1_1 FGA 18 24 26 false
reqbt-1_1 vWA 14 18 false
reqbt-1_1 D3S1358 15 16 18 false
reqbt-1_1 D16S539 9 11 false
reqbt-1_1 D2S1338 19 false
reqbt-1_1 D5S818 9 11 12 false
reqbt-1_1 D7S820 10 11 false
reqbt-1_1 TPOX 11 false
reqbt-1_1 D21S11 29 31 false
reqbt-1_1 D8S1179 10 12 14 false
reqbt-1_1 CSF1PO 12 false
reqbt-1_2 D13S317 12 false
reqbt-1_2 D18S51 22 false
reqbt-1_2 D19S433 12 15 16 false
reqbt-1_2 TH01 6 9 false
reqbt-1_2 FGA 18 24 26 false
reqbt-1_2 vWA 14 18 false
reqbt-1_2 D3S1358 15 16 18 false
reqbt-1_2 D16S539 8 9 11 false
reqbt-1_2 D5S818 9 11 12 false
reqbt-1_2 D7S820 10 11 false
reqbt-1_2 TPOX 8 11 false
reqbt-1_2 D21S11 29 31 false
reqbt-1_2 D8S1179 10 12 14 false
reqbt-1_2 CSF1PO 10 12 false
reqbt-1_3 D13S317 12 false
reqbt-1_3 D19S433 12 15 16 false
reqbt-1_3 TH01 6 9 9.3 false
reqbt-1_3 FGA 24 26 false
reqbt-1_3 vWA 14 18 false
reqbt-1_3 D3S1358 15 17 18 false
reqbt-1_3 D16S539 11 false
reqbt-1_3 D5S818 9 10 11 12 false
reqbt-1_3 D7S820 9 11 false
reqbt-1_3 TPOX 11 false
reqbt-1_3 D21S11 29 31 false
reqbt-1_3 D8S1179 10 12 14 false

1
cjs/.app/database/bug/tmp/reqbt-2-1-settings.json

@ -0,0 +1 @@
{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2-1.csv", "test_name": "reqbt-2-1", "evidence_name": "reqbt-2", "comparison_name": "reqbt-2-1", "num_contributors": 2, "num_replicates": 2}

16
cjs/.app/database/bug/tmp/reqbt-2-1.csv

@ -0,0 +1,16 @@
SampleName,Marker,Allele1,Allele2
reqbt-2-1,D13S317,12,12
reqbt-2-1,D18S51,15,22
reqbt-2-1,D19S433,12,16
reqbt-2-1,TH01,6,9
reqbt-2-1,FGA,24,26
reqbt-2-1,vWA,14,18
reqbt-2-1,D3S1358,15,18
reqbt-2-1,D16S539,9,11
reqbt-2-1,D2S1338,18,19
reqbt-2-1,D5S818,11,12
reqbt-2-1,D7S820,11,11
reqbt-2-1,TPOX,11,11
reqbt-2-1,D21S11,29,31
reqbt-2-1,D8S1179,10,12
reqbt-2-1,CSF1PO,10,12

1
cjs/.app/database/bug/tmp/reqbt-2-2-settings.json

@ -0,0 +1 @@
{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2-2.csv", "test_name": "reqbt-2-2", "evidence_name": "reqbt-2", "comparison_name": "reqbt-2-2", "num_contributors": 2, "num_replicates": 2}

16
cjs/.app/database/bug/tmp/reqbt-2-2.csv

@ -0,0 +1,16 @@
SampleName,Marker,Allele1,Allele2
reqbt-2-2,vWA,17,17
reqbt-2-2,TPOX,8,8
reqbt-2-2,D18S51,12,15
reqbt-2-2,CSF1PO,11,11
reqbt-2-2,D7S820,10,10
reqbt-2-2,D3S1358,14,17
reqbt-2-2,D2S1338,20,20
reqbt-2-2,D8S1179,14,15
reqbt-2-2,D16S539,13,13
reqbt-2-2,D21S11,29,30.2
reqbt-2-2,D13S317,11,12
reqbt-2-2,D19S433,13,15
reqbt-2-2,FGA,22,22
reqbt-2-2,D5S818,10,12
reqbt-2-2,TH01,6,7

29
cjs/.app/database/bug/tmp/reqbt-2.tsv

@ -0,0 +1,29 @@
Sample Name Marker Allele 1 Allele 2 Allele 3 ADO UD1
reqbt-2_1 vWA 17 18 false
reqbt-2_1 TPOX 8 false
reqbt-2_1 D18S51 12 16 false
reqbt-2_1 CSF1PO 10 11 false
reqbt-2_1 D7S820 10 false
reqbt-2_1 D3S1358 14 15 17 false
reqbt-2_1 D2S1338 20 false
reqbt-2_1 D8S1179 12 14 15 false
reqbt-2_1 D16S539 13 false
reqbt-2_1 D21S11 29 31 32.2 false
reqbt-2_1 D13S317 8 11 12 false
reqbt-2_1 D19S433 13 15 false
reqbt-2_1 FGA 22 24 26 false
reqbt-2_1 D5S818 10 11 12 false
reqbt-2_1 TH01 6 7 false
reqbt-2_2 vWA 17 false
reqbt-2_2 TPOX 8 false
reqbt-2_2 D18S51 15 false
reqbt-2_2 D7S820 10 false
reqbt-2_2 D3S1358 14 17 false
reqbt-2_2 D8S1179 14 15 false
reqbt-2_2 D16S539 9 13 false
reqbt-2_2 D21S11 30.2 false
reqbt-2_2 D13S317 11 12 false
reqbt-2_2 D19S433 13 14 15 false
reqbt-2_2 FGA 22 false
reqbt-2_2 D5S818 10 12 false
reqbt-2_2 TH01 6 7 false

1
cjs/.app/database/bug/tmp/reqbt-3-1-settings.json

@ -0,0 +1 @@
{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-3.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-3-1.csv", "test_name": "reqbt-3-1", "evidence_name": "reqbt-3", "comparison_name": "reqbt-3-1", "num_contributors": 2, "num_replicates": 2}

16
cjs/.app/database/bug/tmp/reqbt-3-1.csv

@ -0,0 +1,16 @@
SampleName,Marker,Allele1,Allele2
reqbt-3-1,vWA,15,16
reqbt-3-1,TPOX,9,10
reqbt-3-1,D18S51,12,14
reqbt-3-1,CSF1PO,9,10
reqbt-3-1,D7S820,10,12
reqbt-3-1,D3S1358,16,17
reqbt-3-1,D2S1338,17,18
reqbt-3-1,D8S1179,11,13
reqbt-3-1,D16S539,11,12
reqbt-3-1,D21S11,29,31
reqbt-3-1,D13S317,8,10
reqbt-3-1,D19S433,15.2,17
reqbt-3-1,FGA,19,21
reqbt-3-1,D5S818,9,14
reqbt-3-1,TH01,6,7

31
cjs/.app/database/bug/tmp/reqbt-3.tsv

@ -0,0 +1,31 @@
Sample Name Marker Allele 1 Allele 2 Allele 3 Allele 4 Allele 5 Allele 6 Allele 7 ADO UD1
reqbt-3_1 vWA 14 15 16 false
reqbt-3_1 TPOX 8 9 10 false
reqbt-3_1 D18S51 12 14 false
reqbt-3_1 CSF1PO 9 10 11 12 false
reqbt-3_1 D7S820 10 12 false
reqbt-3_1 D3S1358 15 16 17 18 false
reqbt-3_1 D2S1338 17 false
reqbt-3_1 D8S1179 11 13 14 false
reqbt-3_1 D16S539 11 12 13 false
reqbt-3_1 D21S11 29 30 31 false
reqbt-3_1 D13S317 8 10 11 false
reqbt-3_1 D19S433 15.2 16 17 false
reqbt-3_1 FGA 19 21 false
reqbt-3_1 D5S818 9 13 14 false
reqbt-3_1 TH01 6 7 false
reqbt-3_2 vWA 15 16 false
reqbt-3_2 TPOX 8 9 10 false
reqbt-3_2 D18S51 12 13 14 17 false
reqbt-3_2 CSF1PO 9 10 false
reqbt-3_2 D7S820 9 10 12 false
reqbt-3_2 D3S1358 15 16 17 18 false
reqbt-3_2 D2S1338 17 false
reqbt-3_2 D8S1179 11 13 14 false
reqbt-3_2 D16S539 11 12 false
reqbt-3_2 D21S11 29 30.2 31 false
reqbt-3_2 D13S317 8 10 13 false
reqbt-3_2 D19S433 13.2 14 14.2 15 15.2 16 17 false
reqbt-3_2 FGA 19 21 25 false
reqbt-3_2 D5S818 9 12 14 false
reqbt-3_2 TH01 6 7 false

8
cjs/automate.py

@ -65,9 +65,10 @@ def efm(ctx, database_name):
try:
# pull all pending jobs from the database
job_list = pending_jobs(database_name)
#job_list = pending_jobs(database_name)
#print('hi')
# spawn a size num_workers worker cluster of efm workers
spawn_workers(job_list,
spawn_workers(database_name,
efm_worker,
base_profile,
ctx.obj['num_workers'])
@ -128,10 +129,11 @@ def efm_worker(profile, job):
# write settings file to tmp
write_json(test_file, test_settings)
click.echo('* Starting %s and %s' % (evidence_table.name, comparison_table.name))
try:
out, err = run_process(['../euroformix.headless/R/lrmix.R', test_file], 1800) # 300 seconds = 5 minute timeout
decoded_output = out[0].decode('utf-8')
click.echo(decoded_output)
#click.echo(decoded_output)
if err:
send_error('%s: R Error' % profile['worker_name'],
'Hi,\n\nAn R error occurred:\n %s\n\nSorry about that.\n\nSincerely,\n%s' % (decoded_output, profile['worker_name']))

8
cjs/core/automate.py

@ -12,6 +12,8 @@ from subprocess import TimeoutExpired
import click
# local
from core.database import fill_job_queue
import utils.io
from utils.io import read_json
@ -58,7 +60,7 @@ def generic_worker(fn):
return result
return wrapper
def spawn_workers(job_list, target_fn, base_profile, num_threads, *args, **kwargs):
def spawn_workers(database_name, target_fn, base_profile, num_threads, *args, **kwargs):
# modified from https://docs.python.org/3.7/library/queue.html?highlight=queue#queue.Queue.join
# initialize thread-safe queue
job_queue = queue.Queue()
@ -78,9 +80,13 @@ def spawn_workers(job_list, target_fn, base_profile, num_threads, *args, **kwarg
t.start() # begin worker thread
threads.append(t) # store thread object
# insert jobs into queue
fill_job_queue(database_name, job_queue)
'''
# insert jobs into queue
for job in job_list:
job_queue.put(job)
'''
# block until all tasks are done
job_queue.join()

39
cjs/core/database.py

@ -314,7 +314,15 @@ def pending_jobs(name):
ds = _get_database(name)
pending = ds.select_tables(JobTable, JobTable.event_id == 1)
job_list = []
sess = ds._get_session()
for job in pending:
# TODO: exposing session to get this running for now
next_test = sess.query(TestTable).filter(TestTable.id_ == job.test_id).first()
next_evidence = sess.query(EvidenceTable).filter(EvidenceTable.id_ == next_test.evidence_id).first()
next_comparison = sess.query(ComparisonTable).filter(ComparisonTable.id_ == next_test.comparison_id).first()
job_list.append([job.id_, [next_evidence, next_comparison]])
# this was far too slow for the non-contributors
'''
# use test_id to find the next evidence_id and comparison_id
next_test = ds.first_record(TestTable, TestTable.id_ == job.test_id)
# use the evidence_id to get the next evidence row
@ -324,6 +332,37 @@ def pending_jobs(name):
next_comparison = ds.select_tables(ComparisonTable,
ComparisonTable.id_ == next_test.comparison_id)
job_list.append([job.id_, [next_evidence[0], next_comparison[0]]])
'''
return job_list
except:
# TODO: implement fine-grained handling
raise
def fill_job_queue(name, q):
'''Return a list of all pending jobs.'''
try:
ds = _get_database(name)
pending = ds.select_tables(JobTable, JobTable.event_id == 1)
job_list = []
sess = ds._get_session()
for job in pending:
# TODO: exposing session to get this running for now
next_test = sess.query(TestTable).filter(TestTable.id_ == job.test_id).first()
next_evidence = sess.query(EvidenceTable).filter(EvidenceTable.id_ == next_test.evidence_id).first()
next_comparison = sess.query(ComparisonTable).filter(ComparisonTable.id_ == next_test.comparison_id).first()
q.put([job.id_, [next_evidence, next_comparison]])
# this was far too slow for the non-contributors
'''
# use test_id to find the next evidence_id and comparison_id
next_test = ds.first_record(TestTable, TestTable.id_ == job.test_id)
# use the evidence_id to get the next evidence row
next_evidence = ds.select_tables(EvidenceTable,
EvidenceTable.id_ == next_test.evidence_id)
# use the evidence_id to get the next comparison row
next_comparison = ds.select_tables(ComparisonTable,
ComparisonTable.id_ == next_test.comparison_id)
job_list.append([job.id_, [next_evidence[0], next_comparison[0]]])
'''
return job_list
except:
# TODO: implement fine-grained handling

Loading…
Cancel
Save