#!/usr/bin/env python3 """Score already-paired pilot observations offline; never align or fetch data.""" import argparse import csv import hashlib import io import json from pathlib import Path import re import sys FIELDS = ['observation_id', 'timestamp', 'reference_count', 'reported_value', 'missing_reason'] def integer(value, line, field): if not re.fullmatch(r'[0-9]+', value): raise ValueError(f'CSV line {line}: {field} must be a nonnegative integer or blank') try: return int(value) except ValueError: raise ValueError(f'CSV line {line}: {field} integer is too large') from None def ratio(numerator, denominator): return {'numerator': numerator, 'denominator': denominator, 'value': numerator / denominator if denominator else None} def check(data, mode, space, schedule): if mode not in ('exact', 'capped'): raise ValueError('mode must be exact or capped') if not space.strip() or not schedule.strip(): raise ValueError('space and schedule descriptions must be nonempty') text = data.decode('utf-8-sig') reader = csv.DictReader(io.StringIO(text, newline=''), strict=True) if (reader.fieldnames is None or len(reader.fieldnames) != len(FIELDS) or set(reader.fieldnames) != set(FIELDS)): raise ValueError('CSV must have exactly these columns: ' + ','.join(FIELDS)) rows, seen, errors = [], set(), [] confusion = {'correct_empty': 0, 'false_occupied': 0, 'correct_occupied': 0, 'false_empty': 0} coverage = {'paired': 0, 'missing_reference': 0, 'missing_output': 0, 'missing_both': 0} matches = 0 for raw in reader: line = reader.line_num if None in raw or any(value is None for value in raw.values()): raise ValueError(f'CSV line {line}: wrong number of fields') identity = raw['observation_id'] if not identity.strip() or identity in seen: raise ValueError(f'CSV line {line}: observation_id must be nonempty and unique') seen.add(identity) if not raw['timestamp'].strip(): raise ValueError(f'CSV line {line}: timestamp label must be nonempty') ref_text = raw['reference_count'].strip() reported_text = raw['reported_value'].strip() reference = integer(ref_text, line, 'reference_count') if ref_text else None if mode == 'capped' and reported_text: if reported_text not in ('0', '1', '2', '3+'): raise ValueError(f'CSV line {line}: capped reported_value must be 0,1,2,3+ or blank') reported = reported_text else: reported = integer(reported_text, line, 'reported_value') if reported_text else None status = ('missing_both' if reference is None and reported is None else 'missing_reference' if reference is None else 'missing_output' if reported is None else 'paired') coverage[status] += 1 row = {'input': raw, 'source_line': line, 'pair_status': status} if status == 'paired': occupied = reference > 0 reported_occupied = reported != '0' if mode == 'capped' else reported > 0 classification = ('correct_occupied' if occupied and reported_occupied else 'false_empty' if occupied else 'false_occupied' if reported_occupied else 'correct_empty') confusion[classification] += 1 row['presence_classification'] = classification if mode == 'capped': category = '3+' if reference >= 3 else str(reference) row['reference_category'] = category row['category_match'] = reported == category matches += row['category_match'] else: error = reported - reference row['count_error_reported_minus_reference'] = error errors.append(error) matches += error == 0 rows.append(row) paired = coverage['paired'] empty = confusion['correct_empty'] + confusion['false_occupied'] occupied = confusion['correct_occupied'] + confusion['false_empty'] report = { 'mode': mode, 'space': space, 'schedule_description': schedule, 'input_sha256': hashlib.sha256(data).hexdigest(), 'measurement_unit': 'one listed observation row, not elapsed time', 'listed_rows': len(rows), 'coverage_counts': coverage, 'paired_row_coverage': ratio(paired, len(rows)), 'presence': { 'confusion_counts': confusion, 'agreement': ratio(confusion['correct_empty'] + confusion['correct_occupied'], paired), 'false_occupied_given_reference_empty': ratio(confusion['false_occupied'], empty), 'false_empty_given_reference_occupied': ratio(confusion['false_empty'], occupied)}, 'rows': rows, 'limits': ['Rows must already be paired to the same space and observation time.', 'Missing scheduled rows cannot be discovered from this file.', 'Reference truth, synchronization, sampling adequacy and deployment performance are not verified.', 'Row coverage and agreement are not uptime or duration-weighted accuracy.']} if mode == 'capped': report['capped_category_agreement'] = ratio(matches, paired) else: report['exact_count'] = { 'agreement': ratio(matches, paired), 'mean_absolute_error_people': sum(abs(x) for x in errors) / paired if paired else None, 'mean_signed_error_people': sum(errors) / paired if paired else None} return report def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('csv_file', type=Path) parser.add_argument('--mode', required=True, choices=['exact', 'capped']) parser.add_argument('--space', required=True) parser.add_argument('--schedule', required=True) args = parser.parse_args() try: report = check(args.csv_file.read_bytes(), args.mode, args.space, args.schedule) output = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) except (ValueError, UnicodeError, csv.Error, OSError, OverflowError) as error: print('Input rejected: ' + str(error), file=sys.stderr) return 2 print(output) return 0 if __name__ == '__main__': raise SystemExit(main())