#!/usr/bin/env python
# Copyright 2011-2014 Biomedical Imaging Group Rotterdam, Departments of
# Medical Informatics and Radiology, Erasmus MC, Rotterdam, The Netherlands
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import os
import json
import shellescape
FASTR_LOG_TYPE = 'none'
[docs]def get_parser():
parser = argparse.ArgumentParser()
parser.add_argument('infile', nargs='?',
default=os.path.join(os.getcwd(), '__sink_data__.json'),
metavar='__sink_data__.json', help='result file to cat')
parser.add_argument('--verbose', '-v', dest='verbose', action='store_true',
help='path of the data to print')
parser.add_argument('--sinks', dest='sinks', nargs="*",
help='list results for specified sinks')
parser.add_argument('--nodes', dest='nodes', nargs="*",
help='list results for specified nodes')
parser.add_argument('--samples', dest='samples', nargs="*",
help='list result for all samples')
return parser
[docs]def read_sink_data(infile):
if os.path.isdir(infile):
infile = os.path.join(infile, '__sink_data__.json')
if os.path.isfile(infile) and os.access(infile, os.R_OK):
with open(infile) as sink_data_file:
data = json.load(sink_data_file)
else:
print('ERROR: could not read: {}'.format(infile))
data = None
return data
[docs]def switch_sample_sink(sink_data):
# Switch samples and sinks
samples_overview = dict()
for sink, samples in sink_data.iteritems():
for sample_id, sample in samples.iteritems():
if sample_id not in samples_overview:
samples_overview[sample_id] = dict()
samples_overview[sample_id][sink] = sample
return samples_overview
[docs]def print_sinks(sink_data, sink_ids, verbose):
#filter dict
if sink_ids:
filtered_dict = {sink_id: sink_data[sink_id] for sink_id in sink_ids}
else:
filtered_dict = sink_data
# Print dict
for sink, samples in sorted(filtered_dict.iteritems()):
finished = 0
cancelled = 0
failed_samples_string = []
for sample_id, sample in sorted(samples.iteritems()):
if (sample['status'] == "JobState.cancelled") or (sample['status'] == "JobState.failed"):
cancelled += 1
for error_message in sample['errors']:
failed_samples_string.append(' {}: {}'.format(sample_id, error_message[2]))
elif sample['status'] == "JobState.finished":
finished += 1
print('{} -- {} failed -- {} succeeded'.format(sink, cancelled, finished))
if verbose:
for error_message in failed_samples_string:
print(error_message)
[docs]def print_samples(sink_data, sample_ids, verbose):
samples_overview = switch_sample_sink(sink_data)
# Filter dict
if sample_ids:
filtered_dict = { sample_id: samples_overview[sample_id] for sample_id in sample_ids }
else:
filtered_dict = samples_overview
# Print_sinks(samples_overview)
for sample, sinks in sorted(filtered_dict.iteritems()):
finished = 0
cancelled = 0
failed_sink_string = []
for _, sink in sorted(sinks.iteritems()):
if sink['status'] == "JobState.cancelled" or sink['status'] == "JobState.failed":
cancelled += 1
for error_message in sink['errors']:
failed_sink_string.append(error_message[2])
elif sink['status'] == "JobState.finished":
finished += 1
print('{} -- {} failed -- {} succeeded'.format(sample, cancelled, finished))
if verbose:
for error_message in set(failed_sink_string):
print(error_message)
[docs]def print_job_result(job_file):
from fastr.utils.iohelpers import load_gpickle
from pprint import pprint
job = load_gpickle(job_file)
print('\n\n===== JOB {} ====='.format(job.id))
if hasattr(job, 'network_id'):
print('Network: {}'.format(job.network_id))
if hasattr(job, 'run_id'):
print('Run: {}'.format(job.run_id))
print('Node: {}'.format(job.node_id))
print('Sample index: {}'.format(job.sample_index))
print('Sample id: {}'.format(job.sample_id))
print('Status: {}'.format(job.status))
print('Timestamp: {}'.format(job.timestamp))
print('Job file: {}'.format(job.logfile))
command = job.info_store['process'].get('command', None)
if command is not None:
print('\nCommand:')
print('List representation: {}'.format(command))
printable_command = []
for item in command:
printable_command.append(shellescape.quote(item))
print('String representation: {}'.format(' '.join(printable_command)))
print('\nOutput data:')
pprint(job.output_data)
if 'errors' in job.info_store and isinstance(job.info_store['errors'], list):
print('\n----- ERRORS -----')
for job_error in job.info_store['errors']:
print('- {e[0]}: {e[1]} ({e[2]}:{e[3]})'.format(e=job_error))
print('------------------')
if 'process' in job.info_store:
if 'stdout' in job.info_store['process']:
print('\n----- STDOUT -----')
print(job.info_store['process']['stdout'])
print('------------------')
if 'stderr' in job.info_store['process']:
print('\n----- STDERR -----')
print(job.info_store['process']['stderr'])
print('------------------')
[docs]def print_sample_sink(sink_data, dirname, sample_sink_tuples, verbose):
for sink, sample in sample_sink_tuples:
print('Tracing errors for sample {} from sink {}'.format(sample, sink))
errors = sink_data[sink][sample]['errors']
for error in errors:
error_string = error[0].split(' ')[-1]
node = error_string.split('___')
result_pickle = os.path.join(dirname, node[1], sample, '__fastr_result__.pickle.gz')
print_job_result(result_pickle)
[docs]def print_sample_node(sink_data, dirname, sample_node_tuples, verbose):
for sample, node in sample_node_tuples:
result_pickle = os.path.join(dirname, node, sample, '__fastr_result__.pickle.gz')
print_job_result(result_pickle)
[docs]def main(args=None, unknown_args=None):
"""
Trace samples/sinks from a run
"""
if args is None and unknown_args is None:
# No arguments were parsed yet, parse them now
parser = get_parser()
args, unknown_args = parser.parse_known_args()
sink_data = read_sink_data(args.infile)
if not sink_data:
exit(1)
if args.sinks is not None and args.samples is not None:
if len(args.sinks) == len(args.samples):
sample_sink_tuples = zip(args.sinks, args.samples)
print_sample_sink(sink_data, os.path.dirname(args.infile), sample_sink_tuples, args.verbose)
else:
print("ERROR nr of sinks does not match number of samples")
exit(1)
elif args.nodes is not None and args.samples is not None:
if len(args.nodes) == len(args.samples):
sample_node_tuples = zip(args.samples, args.nodes)
print_sample_node(sink_data, os.path.dirname(args.infile), sample_node_tuples, args.verbose)
else:
print("ERROR nr of sinks does not match number of samples")
exit(1)
elif args.sinks is not None:
print_sinks(sink_data, args.sinks, args.verbose)
elif args.samples is not None:
print_samples(sink_data, args.samples, args.verbose)
else:
parser.print_help()
if __name__ == '__main__':
main()