From ad150e70180baa13207cba946e41317b1415494a Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Fri, 31 Jul 2020 17:51:22 -0700 Subject: [PATCH] Update results csv file rank/diff script and small validate script tweak for batch validation --- results/generate_csv_results.py | 90 ++++++++++++++++++++------------- validate.py | 2 +- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/results/generate_csv_results.py b/results/generate_csv_results.py index 861e6ab3..04cf710a 100644 --- a/results/generate_csv_results.py +++ b/results/generate_csv_results.py @@ -1,57 +1,75 @@ import numpy as np import pandas as pd + results = { - 'results-imagenet.csv': pd.read_csv('results-imagenet.csv'), - 'results-imagenetv2-matched-frequency.csv': pd.read_csv('results-imagenetv2-matched-frequency.csv'), - 'results-sketch.csv': pd.read_csv('results-sketch.csv'), - 'results-imagenet-a.csv': pd.read_csv('results-imagenet-a.csv'), - 'results-imagenet-r.csv': pd.read_csv('results-imagenet-r.csv'), - 'results-imagenet-real.csv': pd.read_csv('results-imagenet-real.csv'), + 'results-imagenet.csv': [ + 'results-imagenet-real.csv', + 'results-imagenetv2-matched-frequency.csv', + 'results-sketch.csv' + ], + 'results-imagenet-a-clean.csv': [ + 'results-imagenet-a.csv', + ], + 'results-imagenet-r-clean.csv': [ + 'results-imagenet-r.csv', + ], } -def diff(csv_file): - base_models = results['results-imagenet.csv']['model'].values - csv_models = results[csv_file]['model'].values +def diff(base_df, test_csv): + base_models = base_df['model'].values + test_df = pd.read_csv(test_csv) + test_models = test_df['model'].values - rank_diff = np.zeros_like(csv_models, dtype='object') - top1_diff = np.zeros_like(csv_models, dtype='object') - top5_diff = np.zeros_like(csv_models, dtype='object') + rank_diff = np.zeros_like(test_models, dtype='object') + top1_diff = np.zeros_like(test_models, dtype='object') + top5_diff = np.zeros_like(test_models, dtype='object') - for rank, model in enumerate(csv_models): + for rank, model in enumerate(test_models): if model in base_models: - base_rank = int(np.where(base_models==model)[0]) - top1_d = results[csv_file]['top1'][rank]-results['results-imagenet.csv']['top1'][base_rank] - top5_d = results[csv_file]['top5'][rank]-results['results-imagenet.csv']['top5'][base_rank] + base_rank = int(np.where(base_models == model)[0]) + top1_d = test_df['top1'][rank] - base_df['top1'][base_rank] + top5_d = test_df['top5'][rank] - base_df['top5'][base_rank] # rank_diff - if rank == base_rank: rank_diff[rank] = f'=' - elif rank > base_rank: rank_diff[rank] = f'-{rank-base_rank}' - else: rank_diff[rank] = f'+{base_rank-rank}' + if rank == base_rank: + rank_diff[rank] = f'0' + elif rank > base_rank: + rank_diff[rank] = f'-{rank - base_rank}' + else: + rank_diff[rank] = f'+{base_rank - rank}' # top1_diff - if top1_d >= .0: top1_diff[rank] = f'+{top1_d:.3f}' - else : top1_diff[rank] = f'-{abs(top1_d):.3f}' + if top1_d >= .0: + top1_diff[rank] = f'+{top1_d:.3f}' + else: + top1_diff[rank] = f'-{abs(top1_d):.3f}' # top5_diff - if top5_d >= .0: top5_diff[rank] = f'+{top5_d:.3f}' - else : top5_diff[rank] = f'-{abs(top5_d):.3f}' + if top5_d >= .0: + top5_diff[rank] = f'+{top5_d:.3f}' + else: + top5_diff[rank] = f'-{abs(top5_d):.3f}' else: - rank_diff[rank] = 'X' - top1_diff[rank] = 'X' - top5_diff[rank] = 'X' - - results[csv_file]['rank_diff'] = rank_diff - results[csv_file]['top1_diff'] = top1_diff - results[csv_file]['top5_diff'] = top5_diff - - results[csv_file]['param_count'] = results[csv_file]['param_count'].map('{:,.2f}'.format) + rank_diff[rank] = '' + top1_diff[rank] = '' + top5_diff[rank] = '' + + test_df['top1_diff'] = top1_diff + test_df['top5_diff'] = top5_diff + test_df['rank_diff'] = rank_diff - results[csv_file].to_csv(csv_file, index=False, float_format='%.3f') + test_df['param_count'] = test_df['param_count'].map('{:,.2f}'.format) + test_df.sort_values('top1', ascending=False, inplace=True) + test_df.to_csv(test_csv, index=False, float_format='%.3f') -for csv_file in results: - if csv_file != 'results-imagenet.csv': - diff(csv_file) +for base_results, test_results in results.items(): + base_df = pd.read_csv(base_results) + base_df.sort_values('top1', ascending=False, inplace=True) + for test_csv in test_results: + diff(base_df, test_csv) + base_df['param_count'] = base_df['param_count'].map('{:,.2f}'.format) + base_df.to_csv(base_results, index=False, float_format='%.3f') diff --git a/validate.py b/validate.py index 29ef7d5e..b0d411c5 100755 --- a/validate.py +++ b/validate.py @@ -271,6 +271,7 @@ def main(): result = OrderedDict(model=args.model) r = {} while not r and batch_size >= args.num_gpu: + torch.cuda.empty_cache() try: args.batch_size = batch_size print('Validating with batch size: %d' % args.batch_size) @@ -281,7 +282,6 @@ def main(): raise e batch_size = max(batch_size // 2, args.num_gpu) print("Validation failed, reducing batch size by 50%") - torch.cuda.empty_cache() result.update(r) if args.checkpoint: result['checkpoint'] = args.checkpoint