fixed job queue issue with being slow. Still has speed issue, but fills while threads work

cjs/.app/database/bug/output/test/qualitative_reqbt-1-1.csv

@@ -0,0 +1,2 @@
+"EvidFile" "POI" "log10LR"
+"reqbt-1" "reqbt-1-1" "9.72328273603544e-20"

cjs/.app/database/bug/output/test/qualitative_reqbt-1-2.csv

@@ -0,0 +1,2 @@
+"EvidFile" "POI" "log10LR"
+"reqbt-1" "reqbt-1-2" "77410906044.1592"

cjs/.app/database/bug/output/test/qualitative_reqbt-2-2.csv

@@ -0,0 +1,2 @@
+"EvidFile" "POI" "log10LR"
+"reqbt-2" "reqbt-2-2" "711627443.60097"

cjs/.app/database/bug/tmp/reqbt-1-1-settings.json

@@ -0,0 +1 @@
+{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1-1.csv", "test_name": "reqbt-1-1", "evidence_name": "reqbt-1", "comparison_name": "reqbt-1-1", "num_contributors": 2, "num_replicates": 3}

cjs/.app/database/bug/tmp/reqbt-1-1.csv

@@ -0,0 +1,16 @@
+SampleName,Marker,Allele1,Allele2
+reqbt-1-1,D13S317,11,11
+reqbt-1-1,D18S51,15,24
+reqbt-1-1,D19S433,13,15
+reqbt-1-1,TH01,6,9
+reqbt-1-1,FGA,18,21
+reqbt-1-1,vWA,17,19
+reqbt-1-1,D3S1358,16,17
+reqbt-1-1,D16S539,11,11
+reqbt-1-1,D2S1338,20,20
+reqbt-1-1,D5S818,9,11
+reqbt-1-1,D7S820,10,12
+reqbt-1-1,TPOX,9,11
+reqbt-1-1,D21S11,29,29
+reqbt-1-1,D8S1179,14,15
+reqbt-1-1,CSF1PO,10,10

cjs/.app/database/bug/tmp/reqbt-1-2-settings.json

@@ -0,0 +1 @@
+{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-1-2.csv", "test_name": "reqbt-1-2", "evidence_name": "reqbt-1", "comparison_name": "reqbt-1-2", "num_contributors": 2, "num_replicates": 3}

cjs/.app/database/bug/tmp/reqbt-1-2.csv

@@ -0,0 +1,16 @@
+SampleName,Marker,Allele1,Allele2
+reqbt-1-2,D13S317,12,12
+reqbt-1-2,D18S51,15,22
+reqbt-1-2,D19S433,12,16
+reqbt-1-2,TH01,6,9
+reqbt-1-2,FGA,24,26
+reqbt-1-2,vWA,14,18
+reqbt-1-2,D3S1358,15,18
+reqbt-1-2,D16S539,9,11
+reqbt-1-2,D2S1338,18,19
+reqbt-1-2,D5S818,11,12
+reqbt-1-2,D7S820,11,11
+reqbt-1-2,TPOX,11,11
+reqbt-1-2,D21S11,29,31
+reqbt-1-2,D8S1179,10,12
+reqbt-1-2,CSF1PO,10,12

cjs/.app/database/bug/tmp/reqbt-1.tsv

@@ -0,0 +1,42 @@
+Sample Name	Marker	Allele 1	Allele 2	Allele 3	Allele 4	ADO	UD1	
+reqbt-1_1	D13S317	12				false		
+reqbt-1_1	D18S51	15	17	22		false		
+reqbt-1_1	D19S433	12	13	14	16	false		
+reqbt-1_1	TH01	6	9			false		
+reqbt-1_1	FGA	18	24	26		false		
+reqbt-1_1	vWA	14	18			false		
+reqbt-1_1	D3S1358	15	16	18		false		
+reqbt-1_1	D16S539	9	11			false		
+reqbt-1_1	D2S1338	19				false		
+reqbt-1_1	D5S818	9	11	12		false		
+reqbt-1_1	D7S820	10	11			false		
+reqbt-1_1	TPOX	11				false		
+reqbt-1_1	D21S11	29	31			false		
+reqbt-1_1	D8S1179	10	12	14		false		
+reqbt-1_1	CSF1PO	12				false		
+reqbt-1_2	D13S317	12				false		
+reqbt-1_2	D18S51	22				false		
+reqbt-1_2	D19S433	12	15	16		false		
+reqbt-1_2	TH01	6	9			false		
+reqbt-1_2	FGA	18	24	26		false		
+reqbt-1_2	vWA	14	18			false		
+reqbt-1_2	D3S1358	15	16	18		false		
+reqbt-1_2	D16S539	8	9	11		false		
+reqbt-1_2	D5S818	9	11	12		false		
+reqbt-1_2	D7S820	10	11			false		
+reqbt-1_2	TPOX	8	11			false		
+reqbt-1_2	D21S11	29	31			false		
+reqbt-1_2	D8S1179	10	12	14		false		
+reqbt-1_2	CSF1PO	10	12			false		
+reqbt-1_3	D13S317	12				false		
+reqbt-1_3	D19S433	12	15	16		false		
+reqbt-1_3	TH01	6	9	9.3		false		
+reqbt-1_3	FGA	24	26			false		
+reqbt-1_3	vWA	14	18			false		
+reqbt-1_3	D3S1358	15	17	18		false		
+reqbt-1_3	D16S539	11				false		
+reqbt-1_3	D5S818	9	10	11	12	false		
+reqbt-1_3	D7S820	9	11			false		
+reqbt-1_3	TPOX	11				false		
+reqbt-1_3	D21S11	29	31			false		
+reqbt-1_3	D8S1179	10	12	14		false		

cjs/.app/database/bug/tmp/reqbt-2-1-settings.json

@@ -0,0 +1 @@
+{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2-1.csv", "test_name": "reqbt-2-1", "evidence_name": "reqbt-2", "comparison_name": "reqbt-2-1", "num_contributors": 2, "num_replicates": 2}

cjs/.app/database/bug/tmp/reqbt-2-1.csv

@@ -0,0 +1,16 @@
+SampleName,Marker,Allele1,Allele2
+reqbt-2-1,D13S317,12,12
+reqbt-2-1,D18S51,15,22
+reqbt-2-1,D19S433,12,16
+reqbt-2-1,TH01,6,9
+reqbt-2-1,FGA,24,26
+reqbt-2-1,vWA,14,18
+reqbt-2-1,D3S1358,15,18
+reqbt-2-1,D16S539,9,11
+reqbt-2-1,D2S1338,18,19
+reqbt-2-1,D5S818,11,12
+reqbt-2-1,D7S820,11,11
+reqbt-2-1,TPOX,11,11
+reqbt-2-1,D21S11,29,31
+reqbt-2-1,D8S1179,10,12
+reqbt-2-1,CSF1PO,10,12

cjs/.app/database/bug/tmp/reqbt-2-2-settings.json

@@ -0,0 +1 @@
+{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-2-2.csv", "test_name": "reqbt-2-2", "evidence_name": "reqbt-2", "comparison_name": "reqbt-2-2", "num_contributors": 2, "num_replicates": 2}

cjs/.app/database/bug/tmp/reqbt-2-2.csv

@@ -0,0 +1,16 @@
+SampleName,Marker,Allele1,Allele2
+reqbt-2-2,vWA,17,17
+reqbt-2-2,TPOX,8,8
+reqbt-2-2,D18S51,12,15
+reqbt-2-2,CSF1PO,11,11
+reqbt-2-2,D7S820,10,10
+reqbt-2-2,D3S1358,14,17
+reqbt-2-2,D2S1338,20,20
+reqbt-2-2,D8S1179,14,15
+reqbt-2-2,D16S539,13,13
+reqbt-2-2,D21S11,29,30.2
+reqbt-2-2,D13S317,11,12
+reqbt-2-2,D19S433,13,15
+reqbt-2-2,FGA,22,22
+reqbt-2-2,D5S818,10,12
+reqbt-2-2,TH01,6,7

cjs/.app/database/bug/tmp/reqbt-2.tsv

@@ -0,0 +1,29 @@
+Sample Name	Marker	Allele 1	Allele 2	Allele 3	ADO	UD1	
+reqbt-2_1	vWA	17	18		false		
+reqbt-2_1	TPOX	8			false		
+reqbt-2_1	D18S51	12	16		false		
+reqbt-2_1	CSF1PO	10	11		false		
+reqbt-2_1	D7S820	10			false		
+reqbt-2_1	D3S1358	14	15	17	false		
+reqbt-2_1	D2S1338	20			false		
+reqbt-2_1	D8S1179	12	14	15	false		
+reqbt-2_1	D16S539	13			false		
+reqbt-2_1	D21S11	29	31	32.2	false		
+reqbt-2_1	D13S317	8	11	12	false		
+reqbt-2_1	D19S433	13	15		false		
+reqbt-2_1	FGA	22	24	26	false		
+reqbt-2_1	D5S818	10	11	12	false		
+reqbt-2_1	TH01	6	7		false		
+reqbt-2_2	vWA	17			false		
+reqbt-2_2	TPOX	8			false		
+reqbt-2_2	D18S51	15			false		
+reqbt-2_2	D7S820	10			false		
+reqbt-2_2	D3S1358	14	17		false		
+reqbt-2_2	D8S1179	14	15		false		
+reqbt-2_2	D16S539	9	13		false		
+reqbt-2_2	D21S11	30.2			false		
+reqbt-2_2	D13S317	11	12		false		
+reqbt-2_2	D19S433	13	14	15	false		
+reqbt-2_2	FGA	22			false		
+reqbt-2_2	D5S818	10	12		false		
+reqbt-2_2	TH01	6	7		false		

cjs/.app/database/bug/tmp/reqbt-3-1-settings.json

@@ -0,0 +1 @@
+{"resource_dir": "/home/csguest/Desktop/cjs2/cjs/resources", "working_dir": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug", "batch_dir": "test", "evidence_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-3.tsv", "comparison_file": "/home/csguest/Desktop/cjs2/cjs/.app/database/bug/tmp/reqbt-3-1.csv", "test_name": "reqbt-3-1", "evidence_name": "reqbt-3", "comparison_name": "reqbt-3-1", "num_contributors": 2, "num_replicates": 2}

cjs/.app/database/bug/tmp/reqbt-3-1.csv

@@ -0,0 +1,16 @@
+SampleName,Marker,Allele1,Allele2
+reqbt-3-1,vWA,15,16
+reqbt-3-1,TPOX,9,10
+reqbt-3-1,D18S51,12,14
+reqbt-3-1,CSF1PO,9,10
+reqbt-3-1,D7S820,10,12
+reqbt-3-1,D3S1358,16,17
+reqbt-3-1,D2S1338,17,18
+reqbt-3-1,D8S1179,11,13
+reqbt-3-1,D16S539,11,12
+reqbt-3-1,D21S11,29,31
+reqbt-3-1,D13S317,8,10
+reqbt-3-1,D19S433,15.2,17
+reqbt-3-1,FGA,19,21
+reqbt-3-1,D5S818,9,14
+reqbt-3-1,TH01,6,7

cjs/.app/database/bug/tmp/reqbt-3.tsv

@@ -0,0 +1,31 @@
+Sample Name	Marker	Allele 1	Allele 2	Allele 3	Allele 4	Allele 5	Allele 6	Allele 7	ADO	UD1	
+reqbt-3_1	vWA	14	15	16					false		
+reqbt-3_1	TPOX	8	9	10					false		
+reqbt-3_1	D18S51	12	14						false		
+reqbt-3_1	CSF1PO	9	10	11	12				false		
+reqbt-3_1	D7S820	10	12						false		
+reqbt-3_1	D3S1358	15	16	17	18				false		
+reqbt-3_1	D2S1338	17							false		
+reqbt-3_1	D8S1179	11	13	14					false		
+reqbt-3_1	D16S539	11	12	13					false		
+reqbt-3_1	D21S11	29	30	31					false		
+reqbt-3_1	D13S317	8	10	11					false		
+reqbt-3_1	D19S433	15.2	16	17					false		
+reqbt-3_1	FGA	19	21						false		
+reqbt-3_1	D5S818	9	13	14					false		
+reqbt-3_1	TH01	6	7						false		
+reqbt-3_2	vWA	15	16						false		
+reqbt-3_2	TPOX	8	9	10					false		
+reqbt-3_2	D18S51	12	13	14	17				false		
+reqbt-3_2	CSF1PO	9	10						false		
+reqbt-3_2	D7S820	9	10	12					false		
+reqbt-3_2	D3S1358	15	16	17	18				false		
+reqbt-3_2	D2S1338	17							false		
+reqbt-3_2	D8S1179	11	13	14					false		
+reqbt-3_2	D16S539	11	12						false		
+reqbt-3_2	D21S11	29	30.2	31					false		
+reqbt-3_2	D13S317	8	10	13					false		
+reqbt-3_2	D19S433	13.2	14	14.2	15	15.2	16	17	false		
+reqbt-3_2	FGA	19	21	25					false		
+reqbt-3_2	D5S818	9	12	14					false		
+reqbt-3_2	TH01	6	7						false		

cjs/automate.py

@@ -65,9 +65,10 @@ def efm(ctx, database_name):
 
     try:
         # pull all pending jobs from the database
-        job_list = pending_jobs(database_name)
+        #job_list = pending_jobs(database_name)
+        #print('hi')
         # spawn a size num_workers worker cluster of efm workers 
-        spawn_workers(job_list, 
+        spawn_workers(database_name,
                       efm_worker, 
                       base_profile,
                       ctx.obj['num_workers'])
@@ -128,10 +129,11 @@ def efm_worker(profile, job):
         # write settings file to tmp
         write_json(test_file, test_settings)
                            
+        click.echo('* Starting %s and %s' % (evidence_table.name, comparison_table.name))
         try:
             out, err = run_process(['../euroformix.headless/R/lrmix.R', test_file], 1800) # 300 seconds = 5 minute timeout
             decoded_output = out[0].decode('utf-8')
-            click.echo(decoded_output)
+            #click.echo(decoded_output)
             if err:
                 send_error('%s: R Error' % profile['worker_name'],
                            'Hi,\n\nAn R error occurred:\n %s\n\nSorry about that.\n\nSincerely,\n%s' % (decoded_output, profile['worker_name']))

cjs/core/automate.py

@@ -12,6 +12,8 @@ from subprocess import TimeoutExpired
 import click
 
 # local
+from core.database import fill_job_queue
+
 import utils.io
 from utils.io import read_json
 
@@ -58,7 +60,7 @@ def generic_worker(fn):
         return result
     return wrapper
 
-def spawn_workers(job_list, target_fn, base_profile, num_threads, *args, **kwargs):
+def spawn_workers(database_name, target_fn, base_profile, num_threads, *args, **kwargs):
     # modified from https://docs.python.org/3.7/library/queue.html?highlight=queue#queue.Queue.join
     # initialize thread-safe queue
     job_queue = queue.Queue()
@@ -78,9 +80,13 @@ def spawn_workers(job_list, target_fn, base_profile, num_threads, *args, **kwarg
         t.start() # begin worker thread
         threads.append(t) # store thread object
 
+    # insert jobs into queue
+    fill_job_queue(database_name, job_queue)
+    '''
     # insert jobs into queue
     for job in job_list:
         job_queue.put(job)
+    '''
 
     # block until all tasks are done
     job_queue.join()

cjs/core/database.py

@@ -314,7 +314,15 @@ def pending_jobs(name):
         ds = _get_database(name)
         pending = ds.select_tables(JobTable, JobTable.event_id == 1)
         job_list = []
+        sess = ds._get_session()
         for job in pending:
+            # TODO: exposing session to get this running for now
+            next_test = sess.query(TestTable).filter(TestTable.id_ == job.test_id).first()
+            next_evidence = sess.query(EvidenceTable).filter(EvidenceTable.id_ == next_test.evidence_id).first()
+            next_comparison = sess.query(ComparisonTable).filter(ComparisonTable.id_ == next_test.comparison_id).first()
+            job_list.append([job.id_, [next_evidence, next_comparison]])
+            # this was far too slow for the non-contributors
+            '''
             # use test_id to find the next evidence_id and comparison_id
             next_test = ds.first_record(TestTable, TestTable.id_ == job.test_id)
             # use the evidence_id to get the next evidence row
@@ -324,6 +332,37 @@ def pending_jobs(name):
             next_comparison = ds.select_tables(ComparisonTable,
                                             ComparisonTable.id_ == next_test.comparison_id)
             job_list.append([job.id_, [next_evidence[0], next_comparison[0]]])
+            ''' 
+        return job_list
+    except:
+        # TODO: implement fine-grained handling
+        raise
+
+def fill_job_queue(name, q):
+    '''Return a list of all pending jobs.'''
+    try:
+        ds = _get_database(name)
+        pending = ds.select_tables(JobTable, JobTable.event_id == 1)
+        job_list = []
+        sess = ds._get_session()
+        for job in pending:
+            # TODO: exposing session to get this running for now
+            next_test = sess.query(TestTable).filter(TestTable.id_ == job.test_id).first()
+            next_evidence = sess.query(EvidenceTable).filter(EvidenceTable.id_ == next_test.evidence_id).first()
+            next_comparison = sess.query(ComparisonTable).filter(ComparisonTable.id_ == next_test.comparison_id).first()
+            q.put([job.id_, [next_evidence, next_comparison]])
+            # this was far too slow for the non-contributors
+            '''
+            # use test_id to find the next evidence_id and comparison_id
+            next_test = ds.first_record(TestTable, TestTable.id_ == job.test_id)
+            # use the evidence_id to get the next evidence row
+            next_evidence = ds.select_tables(EvidenceTable,
+                                            EvidenceTable.id_ == next_test.evidence_id)
+            # use the evidence_id to get the next comparison row
+            next_comparison = ds.select_tables(ComparisonTable,
+                                            ComparisonTable.id_ == next_test.comparison_id)
+            job_list.append([job.id_, [next_evidence[0], next_comparison[0]]])
+            ''' 
         return job_list
     except:
         # TODO: implement fine-grained handling