Browse Source

patched filename clobbering, result count in status, duplicate bug in result table

cjs3
Stephen Lorenz 3 years ago
parent
commit
7a36c9d16b
  1. 7
      cjs/cjs/automate.py
  2. 19
      cjs/cjs/core/batch.py
  3. 2
      cjs/cjs/fmt/efm.py
  4. 1
      cjsd/cjsd/resources/automate.py
  5. 1
      cjsd/cjsd/resources/database.py
  6. 5
      run_cjs

7
cjs/cjs/automate.py

@ -191,7 +191,7 @@ def efm_qualitative(state, job):
# write settings file to tmp
write_json(test_file, test_settings)
click.echo(' Worker: Starting %s and %s (%s)' % (
click.echo(' Worker: %s-%s-%s: Starting job' % (
evidence_data['name'],
comparison_data['name'],
ethnicity
@ -216,10 +216,9 @@ def efm_qualitative(state, job):
except Exception as e:
print(' Worker: An unexpected error has occured (%s, %s)' % (evidence_data['name'], comparison_data['name']))
status['event'] = 4
state.output_queue.put(status)
raise
break
# signal the foreman of the status
print(' Worker: End of job (%s, %s)' % (evidence_data['name'], comparison_data['name']))
click.echo(' Worker: %s-%s-%s: Finished job' % (evidence_data['name'], comparison_data['name'], ethnicity))
state.output_queue.put(status)

19
cjs/cjs/core/batch.py

@ -44,9 +44,6 @@ class FileSystemScanner(PatternMatchingEventHandler):
def on_created(self, event):
self.pending_files.put(event.src_path)
def on_modified(self, event):
self.pending_files.put(event.src_path)
def send_error(subject, message):
send_email('resources/mailer.json',
'cucjsoftware@gmail.com',
@ -146,7 +143,7 @@ def spawn_foreman(state, interface, database, scan_dir):
if not hault:
job = interface.next_job(database)
if job == None:
print('Foreman: Received dismissal signal')
print('Foreman: Received dismissal')
else:
print('Foreman: Received %s and %s' % (job['evidence']['name'], job['comparison']['name']))
state.input_queue.put(job)
@ -155,7 +152,19 @@ def spawn_foreman(state, interface, database, scan_dir):
try:
while True:
status = state.output_queue.get(block=False)
print('Foreman: Updating status for %s and %s to %d' % (status['evidence'], status['comparison'], status['event']))
status_lookup = {
3: 'completed',
4: 'failure'
}
status_announcement = 'Foreman: %s: %s: Updating status to %s' % (
status['evidence'],
status['comparison'],
status_lookup[status['event']]
)
print(status_announcement)
interface.update_job(
database,
status['evidence'],

2
cjs/cjs/fmt/efm.py

@ -104,7 +104,7 @@ def any_bad_values(values):
def reqbt_to_settings(evidence_json, comparison_json):
settings = {
'test_name': comparison_json['name'],
'test_name': '%s-%s' % (evidence_json['name'], comparison_json['name']),
'evidence_name': evidence_json['name'],
'comparison_name': comparison_json['name'],
'num_contributors': evidence_json['contributors'],

1
cjsd/cjsd/resources/automate.py

@ -58,7 +58,6 @@ class UpdateJobResource:
event_id = req.context['event']
job = find_job(database_name, evidence_name, comparison_name)
print(job)
set_job_status(database_name, job.id_, event_id)
except IntegrityError:
print('Unable to find %s and %s' % (evidence_name, comparison_name))

1
cjsd/cjsd/resources/database.py

@ -310,6 +310,7 @@ class StatusResource:
resp.media['comparison_count'] = get_count(ComparisonTable)
resp.media['test_count'] = get_count(TestTable)
resp.media['job_count'] = get_count(JobTable)
resp.media['result_count'] = get_count(ResultTable)
resp.media['pending_count'] = get_count(JobTable, JobTable.event_id == 1)
resp.media['processing_count'] = get_count(JobTable, JobTable.event_id == 2)
resp.media['complete_count'] = get_count(JobTable, JobTable.event_id == 3)

5
run_cjs

@ -1,6 +1,7 @@
#!/usr/bin/env bash
sleep $(($RANDOM % 60))
sleep 1
source .env/bin/activate
cjs --address 128.153.145.175 --port 80 automate --num_workers 4 efm pilot
cjs automate --num_workers 4 efm debug
Loading…
Cancel
Save