diff --git a/README.md b/README.md index db12729a..18c3cbca 100644 --- a/README.md +++ b/README.md @@ -139,12 +139,21 @@ dataset directory. - **documentation** - Missing or incomplete documentation (e.g. `README.md`). - **integrity** - CSV structure, encoding, and duplicate column detection. - **quality** - Missing values, outliers, and data type consistency. -- **privacy** - PHI patterns (SSN, email, phone numbers), date patterns, and - sensitive file detection. +- **privacy** - PHI patterns (SSN, email, phone numbers), ages above 89, + dates that may be real, and sensitive file detection. + +Dates from the last 200 years up to today are reported as possibly real, since +de-identified datasets usually shift dates (often into the future). This covers +written dates in CSV and text files, and Unix timestamps in columns with +time-like names such as `charttime` or `timestamp`. Dates shifted into that +window still need to be confirmed by the submitter. Set +`ValidationConfig(date_lookback_years=...)` to a number of years from 1 to 1000 +to change the window, or `None` to turn the check off. The terminal shows date +warnings for the first 10 files; the saved report lists every file. CSV exports from the Empatica E4 wearable are recognized, so their headerless layout, empty `tags.csv` files, and Unix timestamps don't produce false errors -or privacy warnings. This doesn't check that timestamps are de-identified. +or phone-number warnings. Their timestamps are checked as dates. **Examples:** diff --git a/physionet/cli.py b/physionet/cli.py index da2d64c3..afb51d2e 100644 --- a/physionet/cli.py +++ b/physionet/cli.py @@ -8,6 +8,9 @@ from physionet.validate import validate_dataset, ValidationConfig +# Issues of one kind (such as date warnings) shown per category in the terminal. +TERMINAL_MAX_PER_CODE = 10 + def main(): """Main entry point for the CLI.""" @@ -263,7 +266,8 @@ def _handle_validate(args): result = validate_dataset(str(dataset_path), config, show_progress=True) print() - print(result.summary()) + # Keep the terminal readable; the saved report below lists every issue. + print(result.summary(max_per_code=TERMINAL_MAX_PER_CODE)) # Save validation report - either to specified path or default location if args.report: diff --git a/physionet/validate/checks/csv_layout.py b/physionet/validate/checks/csv_layout.py index af21c5b8..0a64047b 100644 --- a/physionet/validate/checks/csv_layout.py +++ b/physionet/validate/checks/csv_layout.py @@ -17,6 +17,8 @@ class CsvLayout: columns: Optional[Tuple[str, ...]] = None metadata_rows: int = 0 + # Optional names for the metadata rows, used in messages. + metadata_row_names: Tuple[str, ...] = () allow_empty: bool = False empty_message: Optional[str] = None no_data_message: Optional[str] = None diff --git a/physionet/validate/checks/dates.py b/physionet/validate/checks/dates.py new file mode 100644 index 00000000..f7566784 --- /dev/null +++ b/physionet/validate/checks/dates.py @@ -0,0 +1,162 @@ +"""Find dates and Unix timestamps that fall in a realistic, possibly real, range. + +De-identified datasets usually shift dates, often far into the future. A date +between `lookback_years` ago and today may be a real date, so the privacy check +asks the submitter to confirm it was shifted. Format alone cannot tell shifted +dates from real ones, so findings are warnings to review, not errors. + +Recognized forms: +- Year-first dates: 2023-01-13, 2023/01/13, 2023.01.13, with an optional time. +- Day- or month-first dates with a four-digit year: 13/01/2023, 01-13-2023. +- Month names: 13 Jan 2023, January 13, 2023, Jan 2023. +- Unix seconds (9-10 digits) or milliseconds (12-13 digits), but only in + columns whose name suggests a time (or that a layout names), because IDs and + counters have the same shape. + +Two-digit years, eight-digit compact dates (20230113), times with no date, and +years on their own are not reported. +""" + +import re +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Iterator, Optional + +from physionet.validate.config import validate_lookback_years + + +_MONTH_NAME = ( + r"(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?" + r"|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.?" +) +_MONTHS = {name: number for number, name in enumerate( + ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], start=1 +)} + +_YEAR_FIRST = re.compile(r"(? Optional[date]: + try: + return date(year, month, day) + except ValueError: + return None + + +def _month(name: str) -> int: + return _MONTHS[name.lower()[:3]] + + +def dates_in_text(text: str) -> Iterator[date]: + """Yield every calendar date written in text.""" + if not _MAYBE_WRITTEN_DATE.search(text) or not any(char.isdigit() for char in text): + return + for match in _YEAR_FIRST.finditer(text): + found = _make_date(int(match[1]), int(match[3]), int(match[4])) + if found: + yield found + for match in _YEAR_LAST.finditer(text): + first, second, year = int(match[1]), int(match[3]), int(match[4]) + # Ambiguous order: accept month-first or day-first, whichever is valid. + found = _make_date(year, first, second) or _make_date(year, second, first) + if found: + yield found + if not any(char.isalpha() for char in text): + return + full_date_spans = [] + for match in _DAY_MONTH_YEAR.finditer(text): + found = _make_date(int(match[3]), _month(match[2]), int(match[1])) + if found: + full_date_spans.append(match.span()) + yield found + for match in _MONTH_DAY_YEAR.finditer(text): + found = _make_date(int(match[3]), _month(match[1]), int(match[2])) + if found: + full_date_spans.append(match.span()) + yield found + for match in _MONTH_YEAR.finditer(text): + # "5 May 2019" also contains "May 2019"; count only the full date. + if any(start < match.end() and match.start() < end for start, end in full_date_spans): + continue + found = _make_date(int(match[2]), _month(match[1]), 1) + if found: + yield found + + +def unix_timestamp_date(value: str) -> Optional[date]: + """Return the UTC date of Unix seconds or milliseconds, if value is one.""" + match = _UNIX.match(value.strip()) + if not match: + return None + digits = match[1] + seconds = int(digits) / 1000 if len(digits) > 10 else int(digits) + try: + return datetime.fromtimestamp(seconds, tz=timezone.utc).date() + except (OverflowError, OSError, ValueError): + return None + + +def is_time_like_column(name: Optional[str]) -> bool: + """Guess from a column name whether it holds times (e.g. charttime, ts, date).""" + return bool(name) and bool(_TIME_LIKE_COLUMN.search(name)) + + +@dataclass +class DateRange: + """Earliest and latest realistic dates seen in one place.""" + + earliest: date + latest: date + + def add(self, found: date) -> None: + self.earliest = min(self.earliest, found) + self.latest = max(self.latest, found) + + def describe(self) -> str: + if self.earliest == self.latest: + return self.earliest.isoformat() + return f"{self.earliest.isoformat()} to {self.latest.isoformat()}" + + +class RealisticDateFinder: + """Collect dates between lookback_years ago and today.""" + + def __init__(self, lookback_years: int, today: Optional[date] = None): + validate_lookback_years(lookback_years) + today = today or datetime.now(timezone.utc).date() + self.lookback_years = lookback_years + self.start = date(today.year - lookback_years, 1, 1) + self.end = today + timedelta(days=1) # Allow for time-zone differences. + + def in_range(self, found: date) -> bool: + return self.start <= found <= self.end + + def scan(self, value: str, unix_allowed: bool, ranges: dict, key: str) -> None: + """Record realistic dates in value under key in ranges.""" + candidates = list(dates_in_text(value)) + if unix_allowed: + unix_date = unix_timestamp_date(value) + if unix_date: + candidates.append(unix_date) + for found in candidates: + if not self.in_range(found): + continue + if key in ranges: + ranges[key].add(found) + else: + ranges[key] = DateRange(found, found) diff --git a/physionet/validate/checks/empatica.py b/physionet/validate/checks/empatica.py index 64c9b8d6..1a0fa1ec 100644 --- a/physionet/validate/checks/empatica.py +++ b/physionet/validate/checks/empatica.py @@ -11,6 +11,8 @@ treated as E4 when a recognized sensor or IBI file sits beside it, and only its event_time column is exempt from the phone-number pattern. +Start times and event times are still checked as dates by the privacy check. + File names are matched in any letter case. tests/fixtures/e4_session holds a small synthetic example. """ @@ -101,11 +103,16 @@ def get_e4_csv_layout(file_path: Path) -> Optional[CsvLayout]: """Describe recognized E4 sensor, IBI, and event-tag files for CSV checks.""" columns = e4_sensor_columns(file_path) if columns is not None: - return CsvLayout(columns=columns, metadata_rows=2) + return CsvLayout( + columns=columns, + metadata_rows=2, + metadata_row_names=("start_time", "sample_rate"), + ) if is_e4_ibi_file(file_path): return CsvLayout( columns=_IBI_COLUMNS, metadata_rows=1, + metadata_row_names=("start_time",), no_data_message="No Empatica E4 inter-beat intervals detected (IBI.csv has only its start-time row)", ) if is_e4_tags_file(file_path): diff --git a/physionet/validate/checks/privacy.py b/physionet/validate/checks/privacy.py index 1eb530f0..cbf230c2 100644 --- a/physionet/validate/checks/privacy.py +++ b/physionet/validate/checks/privacy.py @@ -1,14 +1,21 @@ """Privacy and PHI validation checks.""" +import csv import os import re from pathlib import Path -from typing import Optional, Callable +from typing import Callable, Dict, List, Optional, Tuple from physionet.validate.models import CheckResult, ValidationIssue, CheckCategory, Severity from physionet.validate.config import ValidationConfig from physionet.validate.checks.csv_formats import find_files_by_suffix, get_csv_layout from physionet.validate.checks.csv_layout import is_unix_timestamp +from physionet.validate.checks.dates import ( + DateRange, RealisticDateFinder, dates_in_text, is_time_like_column, +) + +# Identifies date warnings so the terminal summary can shorten them. +POSSIBLE_REAL_DATE = "possible-real-date" # Pattern names for better error messages PHI_PATTERN_NAMES = { @@ -58,7 +65,7 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio - PHI pattern detection - Age de-identification - Sensitive configuration files (keys, credentials) - - Date patterns + - Dates within config.date_lookback_years of today, which may be real Args: path: Path to dataset directory @@ -79,6 +86,12 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio pattern_info = [(re.compile(pattern), PHI_PATTERN_NAMES.get(pattern, "unknown pattern")) for pattern in config.phi_patterns] + date_finder = ( + RealisticDateFinder(config.date_lookback_years) + if config.date_lookback_years is not None else None + ) + date_findings: List[Tuple[str, Dict[Optional[str], DateRange]]] = [] + # Check CSV files csv_files = find_files_by_suffix(path, ".csv") for i, csv_file in enumerate(csv_files): @@ -88,7 +101,7 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio if any(p in str(csv_file) for p in config.ignore_patterns): continue - _check_csv_privacy(csv_file, path, config, pattern_info, result) + _check_csv_privacy(csv_file, path, config, pattern_info, result, date_finder, date_findings) # Check text files for PHI text_files = find_files_by_suffix(path, ".txt") @@ -99,11 +112,44 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio if any(p in str(text_file) for p in config.ignore_patterns): continue - _check_text_file_privacy(text_file, path, pattern_info, result, config) + _check_text_file_privacy( + text_file, path, pattern_info, result, config, date_finder, date_findings + ) + + if date_finder: + _report_realistic_dates(date_findings, config.date_lookback_years, result) return result +def _report_realistic_dates( + findings: List[Tuple[str, Dict[Optional[str], DateRange]]], + lookback_years: int, + result: CheckResult, +) -> None: + """Warn about files holding dates that may be real, one warning per file.""" + for relative_path, ranges in findings: + places = [] + for column, date_range in ranges.items(): + where = f"column '{column}': " if column is not None else "" + places.append(f"{where}{date_range.describe()}") + columns = [column for column in ranges if column is not None] + result.issues.append( + ValidationIssue( + severity=Severity.WARNING, + category=CheckCategory.PRIVACY, + file=relative_path, + column=columns[0] if len(columns) == 1 else None, + message=( + f"Dates within the last {lookback_years} years may be real " + f"({'; '.join(places)})" + ), + suggestion="Confirm these dates have been shifted or removed", + code=POSSIBLE_REAL_DATE, + ) + ) + + def _check_sensitive_files(path: Path, config: ValidationConfig, result: CheckResult) -> None: """Check for sensitive configuration files that shouldn't be in the dataset.""" for root, dirs, files in os.walk(path): @@ -167,10 +213,13 @@ def _check_csv_privacy( base_path: Path, config: ValidationConfig, pattern_info: list, - result: CheckResult + result: CheckResult, + date_finder: Optional[RealisticDateFinder] = None, + date_findings: Optional[list] = None, ) -> None: """Check a CSV file for privacy issues.""" relative_path = str(csv_file.relative_to(base_path)) + date_ranges: Dict[Optional[str], DateRange] = {} # Report each pattern at most once per column, including malformed extra fields. phi_columns = {} # {column: set of pattern names} @@ -178,6 +227,15 @@ def _check_csv_privacy( layout = get_csv_layout(csv_file) try: + if date_finder and layout.columns is not None and layout.metadata_rows: + # Headerless layouts may keep start times in rows before the data. + with open(csv_file, "r", encoding="utf-8", newline="") as f: + for index, row in zip(range(layout.metadata_rows), csv.reader(f)): + names = layout.metadata_row_names + label = names[index] if index < len(names) else f"metadata row {index + 1}" + for value in row: + date_finder.scan(value, True, date_ranges, label) + with open(csv_file, "r", encoding="utf-8") as f: reader = layout.dict_reader(f) @@ -223,6 +281,13 @@ def _check_csv_privacy( value_str = str(value).strip() + if date_finder: + unix_allowed = raw_col is not None and ( + raw_col in layout.unix_timestamp_columns + or is_time_like_column(raw_col) + ) + date_finder.scan(value_str, unix_allowed, date_ranges, col) + for pattern, pattern_name in pattern_info: if (pattern_name == "phone number" and raw_col in layout.unix_timestamp_columns @@ -266,6 +331,9 @@ def _check_csv_privacy( ) ) + if date_ranges and date_findings is not None: + date_findings.append((relative_path, date_ranges)) + except Exception as e: result.issues.append( ValidationIssue( @@ -277,10 +345,19 @@ def _check_csv_privacy( ) -def _check_text_file_privacy(text_file: Path, base_path: Path, pattern_info: list, result: CheckResult, config: ValidationConfig) -> None: +def _check_text_file_privacy( + text_file: Path, + base_path: Path, + pattern_info: list, + result: CheckResult, + config: ValidationConfig, + date_finder: Optional[RealisticDateFinder] = None, + date_findings: Optional[list] = None, +) -> None: """Check a text file for privacy issues.""" relative_path = str(text_file.relative_to(base_path)) detected_patterns = set() + date_ranges: Dict[Optional[str], DateRange] = {} try: with open(text_file, "r", encoding="utf-8") as f: @@ -291,6 +368,12 @@ def _check_text_file_privacy(text_file: Path, base_path: Path, pattern_info: lis for pattern, pattern_name in pattern_info: if pattern.search(line): detected_patterns.add(pattern_name) + if date_finder: + # Free text has no column names, so Unix timestamps are not inferred. + date_finder.scan(line, False, date_ranges, None) + + if date_ranges and date_findings is not None: + date_findings.append((relative_path, date_ranges)) # Report once per file with specific patterns found if detected_patterns: diff --git a/physionet/validate/config.py b/physionet/validate/config.py index f596d689..d286e67d 100644 --- a/physionet/validate/config.py +++ b/physionet/validate/config.py @@ -3,6 +3,17 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple +MAX_LOOKBACK_YEARS = 1000 + + +def validate_lookback_years(value) -> None: + """Reject date windows that would contain no dates or cannot be represented.""" + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_LOOKBACK_YEARS: + raise ValueError( + f"date_lookback_years must be a whole number from 1 to {MAX_LOOKBACK_YEARS}, " + f"or None to turn the date check off (got {value!r})" + ) + @dataclass class ValidationConfig: @@ -37,8 +48,15 @@ class ValidationConfig: # Privacy settings allowed_age_max: int = 89 + # Warn about dates from this many years ago up to today, which may be real + # rather than shifted. None turns the date check off. + date_lookback_years: Optional[int] = 200 phi_patterns: List[str] = field(default_factory=lambda: [ r"\b\d{3}-\d{2}-\d{4}\b", # SSN pattern r"\b[\w\.-]+@[\w\.-]+\.\w+\b", # Email pattern r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", # Phone pattern ]) + + def __post_init__(self) -> None: + if self.date_lookback_years is not None: + validate_lookback_years(self.date_lookback_years) diff --git a/physionet/validate/models.py b/physionet/validate/models.py index 6fc105d0..c30bae4f 100644 --- a/physionet/validate/models.py +++ b/physionet/validate/models.py @@ -34,6 +34,8 @@ class ValidationIssue: column: Optional[str] = None value: Optional[str] = None suggestion: Optional[str] = None + # Groups issues of one kind, e.g. so a summary can shorten long runs of them. + code: Optional[str] = None def to_dict(self) -> Dict[str, Any]: """Convert issue to dictionary format.""" @@ -52,6 +54,8 @@ def to_dict(self) -> Dict[str, Any]: result["value"] = self.value if self.suggestion: result["suggestion"] = self.suggestion + if self.code: + result["code"] = self.code return result @@ -126,8 +130,12 @@ def status(self) -> str: return "warning" return "pass" - def summary(self) -> str: - """Generate a human-readable summary.""" + def summary(self, max_per_code: Optional[int] = None) -> str: + """Generate a human-readable summary. + + With max_per_code, show at most that many issues sharing a code in each + category and count the rest. Issues without a code are always shown. + """ # Format timestamp as human-readable try: dt = datetime.fromisoformat(self.timestamp.replace('Z', '+00:00')) @@ -184,12 +192,27 @@ def summary(self) -> str: lines.append(f"{status_icon} {category.value.replace('_', ' ').title()}{issue_summary}") + shown_per_code: Dict[str, int] = {} + hidden_per_code: Dict[str, int] = {} for issue in result.issues: + if issue.code and max_per_code is not None: + if shown_per_code.get(issue.code, 0) >= max_per_code: + hidden_per_code[issue.code] = hidden_per_code.get(issue.code, 0) + 1 + continue + shown_per_code[issue.code] = shown_per_code.get(issue.code, 0) + 1 icon = "✗" if issue.severity == Severity.ERROR else "⚠" - location = f" {issue.file}" - if issue.line: - location += f":{issue.line}" - lines.append(f" {icon}{location} - {issue.message}") + if issue.file: + location = f" {issue.file}" + if issue.line: + location += f":{issue.line}" + lines.append(f" {icon}{location} - {issue.message}") + else: + lines.append(f" {icon} {issue.message}") + for code, hidden in hidden_per_code.items(): + lines.append( + f" … {hidden} more {code} issue{'s' if hidden != 1 else ''} " + f"not shown here; the saved report lists them all" + ) lines.append("") diff --git a/tests/fixtures/e4_session/ACC.csv b/tests/fixtures/e4_session/ACC.csv index 3912e000..409d3e39 100644 --- a/tests/fixtures/e4_session/ACC.csv +++ b/tests/fixtures/e4_session/ACC.csv @@ -1,4 +1,4 @@ -1673588411.000000,1673588411.000000,1673588411.000000 +5680288411.000000,5680288411.000000,5680288411.000000 32.000000,32.000000,32.000000 27,-25,50 28,-24,49 diff --git a/tests/fixtures/e4_session/BVP.csv b/tests/fixtures/e4_session/BVP.csv index 5871955e..9422c9b4 100644 --- a/tests/fixtures/e4_session/BVP.csv +++ b/tests/fixtures/e4_session/BVP.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 64.000000 0.123456 0.234567 diff --git a/tests/fixtures/e4_session/EDA.csv b/tests/fixtures/e4_session/EDA.csv index d2eaa83e..e2a9fdf3 100644 --- a/tests/fixtures/e4_session/EDA.csv +++ b/tests/fixtures/e4_session/EDA.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 4.000000 0.123456 0.234567 diff --git a/tests/fixtures/e4_session/HR.csv b/tests/fixtures/e4_session/HR.csv index cc04ba8a..aad17f98 100644 --- a/tests/fixtures/e4_session/HR.csv +++ b/tests/fixtures/e4_session/HR.csv @@ -1,4 +1,4 @@ -1673588421.000000 +5680288421.000000 1.000000 72.00 72.50 diff --git a/tests/fixtures/e4_session/IBI.csv b/tests/fixtures/e4_session/IBI.csv index aa71cc3d..4cdb509d 100644 --- a/tests/fixtures/e4_session/IBI.csv +++ b/tests/fixtures/e4_session/IBI.csv @@ -1,3 +1,3 @@ -1673588411.000000, IBI +5680288411.000000, IBI 12.343750,0.828125 13.156250,0.812500 diff --git a/tests/fixtures/e4_session/README.md b/tests/fixtures/e4_session/README.md index 396e5bf4..ffe479d6 100644 --- a/tests/fixtures/e4_session/README.md +++ b/tests/fixtures/e4_session/README.md @@ -4,4 +4,5 @@ This tiny, invented export is test data for the PhysioNet validator. It follows the E4 Connect CSV layout: a Unix start time, a sample rate, then signal samples. IBI.csv instead starts with the start time and `IBI`, followed by rows of seconds since the start and inter-beat interval in seconds. -The timestamps and signal values do not describe a real person or recording. +The timestamps are set in 2150, as a date-shifted dataset might be, and the +signal values do not describe a real person or recording. diff --git a/tests/fixtures/e4_session/TEMP.csv b/tests/fixtures/e4_session/TEMP.csv index 85cb5bca..22e5c697 100644 --- a/tests/fixtures/e4_session/TEMP.csv +++ b/tests/fixtures/e4_session/TEMP.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 4.000000 32.500000 32.600000 diff --git a/tests/fixtures/e4_session/tags.csv b/tests/fixtures/e4_session/tags.csv index 1a88b290..0e21b31b 100644 --- a/tests/fixtures/e4_session/tags.csv +++ b/tests/fixtures/e4_session/tags.csv @@ -1,2 +1,2 @@ -1673588411.000000 -1673588422.000000 +5680288411.000000 +5680288422.000000 diff --git a/tests/validate/test_csv_layout.py b/tests/validate/test_csv_layout.py index 0493cad1..8085314f 100644 --- a/tests/validate/test_csv_layout.py +++ b/tests/validate/test_csv_layout.py @@ -10,8 +10,8 @@ def test_custom_headerless_layout_is_used_by_all_csv_checks(tmp_path, monkeypatch): (tmp_path / "signals.csv").write_text( "device export\n" - "1673588411,0.1\n" - "1673588422,\n" + "5680288411,0.1\n" + "5680288422,\n" ) def detect(path): diff --git a/tests/validate/test_dates.py b/tests/validate/test_dates.py new file mode 100644 index 00000000..8fb7fc95 --- /dev/null +++ b/tests/validate/test_dates.py @@ -0,0 +1,213 @@ +"""Dates that may be real are flagged for review; shifted or ambiguous values are not.""" + +import json +import subprocess +import sys +from datetime import date + +import pytest + +from physionet.validate import ValidationConfig, validate_dataset +from physionet.validate.checks import check_privacy +from physionet.validate.models import CheckCategory +from physionet.validate.checks.dates import ( + RealisticDateFinder, dates_in_text, is_time_like_column, unix_timestamp_date, +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("2023-01-13", date(2023, 1, 13)), + ("2023-01-13T08:15:00", date(2023, 1, 13)), + ("2023-01-13 08:15", date(2023, 1, 13)), + ("2023/1/13", date(2023, 1, 13)), + ("13/01/2023", date(2023, 1, 13)), + ("01-13-2023", date(2023, 1, 13)), + ("Admitted 13 Jan 2023 overnight", date(2023, 1, 13)), + ("January 13th, 2023", date(2023, 1, 13)), + ("Sept 2019", date(2019, 9, 1)), + ], +) +def test_written_dates_are_found(text, expected): + assert expected in list(dates_in_text(text)) + + +def test_full_date_is_not_also_counted_as_month_and_year(): + assert list(dates_in_text("seen 5 May 2019 and in June 2020")) == [ + date(2019, 5, 5), date(2020, 6, 1) + ] + + +@pytest.mark.parametrize( + "text", + [ + "0.123456", "13/01/23", "20230113", "08:15:00", "2023", "1.2.3", + "2023-13-45", "maybe 2023", "version 1.10.2", "-25.50", + ], +) +def test_non_dates_are_ignored(text): + assert list(dates_in_text(text)) == [] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("1673588411", date(2023, 1, 13)), + ("1673588411.250000", date(2023, 1, 13)), + ("1673588411000", date(2023, 1, 13)), + ("631152000", date(1990, 1, 1)), + ("12345678", None), + ("0673588411", None), + ("2023-01-13", None), + ], +) +def test_unix_seconds_and_milliseconds(value, expected): + assert unix_timestamp_date(value) == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("charttime", True), ("event_time", True), ("Timestamp", True), ("ts", True), + ("start_ts", True), ("admit_date", True), ("epoch", True), + ("subject_id", False), ("status", False), ("patients", False), (None, False), + ], +) +def test_time_like_column_names(name, expected): + assert is_time_like_column(name) is expected + + +def test_range_runs_from_lookback_years_ago_to_today(): + finder = RealisticDateFinder(200, today=date(2026, 10, 20)) + + assert finder.in_range(date(1826, 1, 1)) + assert finder.in_range(date(2026, 10, 20)) + assert not finder.in_range(date(1825, 12, 31)) + assert not finder.in_range(date(2150, 1, 1)) + + +def _date_warnings(result): + return [issue for issue in result.issues if "Dates within the last" in issue.message] + + +def test_csv_dates_are_reported_once_per_file_with_their_range(tmp_path): + (tmp_path / "events.csv").write_text( + "subject_id,charttime,note\n" + "10000032,2019-03-04 10:00,seen 5 May 2019\n" + "10000032,2019-05-01 08:30,\n" + ) + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert len(warnings) == 1 + assert warnings[0].file == "events.csv" + assert "column 'charttime': 2019-03-04 to 2019-05-01" in warnings[0].message + assert "column 'note': 2019-05-05" in warnings[0].message + + +def test_shifted_future_dates_are_not_reported(tmp_path): + (tmp_path / "events.csv").write_text( + "charttime,epoch\n2180-07-23 10:00,6531840000\n" + ) + + assert _date_warnings(check_privacy(tmp_path, ValidationConfig())) == [] + + +def test_unix_timestamps_need_a_time_like_column(tmp_path): + (tmp_path / "ids.csv").write_text("record_id\n1673588411\n") + (tmp_path / "times.csv").write_text("timestamp_ms\n1673588411000\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [(issue.file, issue.column) for issue in warnings] == [("times.csv", "timestamp_ms")] + + +def test_text_file_dates_are_reported(tmp_path): + (tmp_path / "notes.txt").write_text("Patient admitted on 13 Jan 2023.\nNo other dates.\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [issue.file for issue in warnings] == ["notes.txt"] + assert "2023-01-13" in warnings[0].message + + +def test_e4_start_times_and_event_times_are_reported_as_dates_not_phones(tmp_path): + (tmp_path / "EDA.csv").write_text("1673588411.000000\n4.000000\n0.123456\n") + (tmp_path / "tags.csv").write_text("1673588500.000000\n") + + result = check_privacy(tmp_path, ValidationConfig()) + + assert not any("phone number" in issue.message for issue in result.issues) + messages = {issue.file: issue.message for issue in _date_warnings(result)} + assert "column 'start_time': 2023-01-13" in messages["EDA.csv"] + assert "column 'event_time': 2023-01-13" in messages["tags.csv"] + + +def test_date_check_can_be_turned_off(tmp_path): + (tmp_path / "events.csv").write_text("charttime\n2019-03-04\n") + + result = check_privacy(tmp_path, ValidationConfig(date_lookback_years=None)) + + assert _date_warnings(result) == [] + + +def test_every_file_with_dates_is_kept_and_only_the_terminal_is_shortened(tmp_path): + for number in range(13): + (tmp_path / f"visit_{number:02d}.csv").write_text("charttime\n2019-03-04\n") + (tmp_path / "README.md").write_text("Synthetic visits.\n") + report = tmp_path / "report.json" + + result = validate_dataset( + str(tmp_path), ValidationConfig(check_documentation=False), show_progress=False + ) + warnings = _date_warnings(result.check_results[CheckCategory.PRIVACY]) + assert sorted(issue.file for issue in warnings) == [f"visit_{n:02d}.csv" for n in range(13)] + assert all(issue.to_dict()["code"] == "possible-real-date" for issue in warnings) + + assert result.summary().count("may be real") == 13 + short = result.summary(max_per_code=10) + assert short.count("may be real") == 10 + assert "3 more possible-real-date issues not shown here" in short + + cli = subprocess.run( + [sys.executable, "-m", "physionet", "validate", str(tmp_path), "--report", str(report)], + capture_output=True, text=True, + ) + assert cli.stdout.count("may be real") == 10 + assert "3 more possible-real-date issues" in cli.stdout + issues = json.loads(report.read_text())["checks"]["privacy"]["issues"] + assert len([issue for issue in issues if issue.get("code") == "possible-real-date"]) == 13 + + markdown = tmp_path / "PHYSIONET_REPORT.md" + subprocess.run([sys.executable, "-m", "physionet", "validate", str(tmp_path)], capture_output=True) + assert markdown.read_text().count("may be real") == 13 + + +@pytest.mark.parametrize( + "text", + ["Seen on 13/01/2023.", "Seen on 13/01/2023, then home.", "Seen on 13/01/2023"], +) +def test_sentence_ending_dates_are_reported(tmp_path, text): + (tmp_path / "notes.txt").write_text(text + "\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [issue.file for issue in warnings] == ["notes.txt"] + assert "2023-01-13" in warnings[0].message + + +@pytest.mark.parametrize("text", ["13.01.2023.5", "1.13.01.2023"]) +def test_numbers_extending_a_date_are_not_dates(text): + assert list(dates_in_text(text)) == [] + + +@pytest.mark.parametrize("value", [-1, 0, 1001, 3000, 1.5, "200", True]) +def test_invalid_date_window_is_rejected(value): + with pytest.raises(ValueError, match="date_lookback_years"): + ValidationConfig(date_lookback_years=value) + + +@pytest.mark.parametrize("value", [None, 1, 200, 1000]) +def test_valid_date_windows_are_accepted(value): + assert ValidationConfig(date_lookback_years=value).date_lookback_years == value diff --git a/tests/validate/test_empatica.py b/tests/validate/test_empatica.py index 5d03c9e3..84c30555 100644 --- a/tests/validate/test_empatica.py +++ b/tests/validate/test_empatica.py @@ -11,7 +11,7 @@ from physionet.validate.checks import check_filesystem, check_integrity, check_privacy, check_quality from physionet.validate.checks.csv_layout import is_unix_timestamp from physionet.validate.checks.empatica import is_e4_ibi_file, is_e4_sensor_file -from physionet.validate.models import Severity +from physionet.validate.models import CheckCategory, Severity E4_EXAMPLE = Path(__file__).parent.parent / "fixtures" / "e4_session" @@ -19,13 +19,17 @@ def write_e4_sensor(directory): (directory / "EDA.csv").write_text( - "1673588411.000000\n4.000000\n0.123456\n0.234567\n0.345678\n" + "5680288411.000000\n4.000000\n0.123456\n0.234567\n0.345678\n" ) -@pytest.mark.parametrize("timestamp", ["1673588411", "631152000", "5680281600"]) +@pytest.mark.parametrize( + ("timestamp", "realistic"), + [("1673588411", True), ("631152000", True), ("5680281600", False)], +) @pytest.mark.parametrize("empty_tags", [True, False]) -def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, empty_tags): +def test_integer_and_shifted_e4_timestamps(tmp_path, timestamp, realistic, empty_tags): + """Format checks pass either way; only dates up to today get a date warning.""" (tmp_path / "ACC.csv").write_text( f"{timestamp},{timestamp},{timestamp}\n" "32,32,32\n" @@ -41,8 +45,12 @@ def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, ) assert result.total_errors == 0 - assert result.total_warnings == 0 assert result.total_info == (1 if empty_tags else 0) + warnings = result.check_results[CheckCategory.PRIVACY].issues + expected_files = {"ACC.csv", "EDA.csv"} | (set() if empty_tags else {"tags.csv"}) + assert result.total_warnings == len(warnings) + assert {issue.file for issue in warnings} == (expected_files if realistic else set()) + assert all("may be real" in issue.message for issue in warnings) @pytest.mark.parametrize( @@ -50,7 +58,7 @@ def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, [ ("631152000", True), ("5680281600.123456", True), - ("1673588411", True), + ("5680288411", True), ("63115200", False), ("56802816000", False), ("-631152000", False), @@ -66,7 +74,7 @@ def test_unix_timestamp_shape(timestamp, expected): def test_e4_headerless_sensor_files_do_not_trigger_csv_structure_errors(tmp_path): write_e4_sensor(tmp_path) (tmp_path / "ACC.csv").write_text( - "1673588411.000000,1673588411.000000,1673588411.000000\n" + "5680288411.000000,5680288411.000000,5680288411.000000\n" "32.000000,32.000000,32.000000\n" "27,-25,50\n27,-24,50\n" ) @@ -79,7 +87,7 @@ def test_e4_headerless_sensor_files_do_not_trigger_csv_structure_errors(tmp_path def test_e4_tags_timestamps_are_not_reported_as_phone_numbers(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411.000000\n1673588422.000000\n") + (tmp_path / "tags.csv").write_text("5680288411.000000\n5680288422.000000\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -115,7 +123,7 @@ def test_e4_sensor_still_reports_malformed_rows(tmp_path): def test_e4_sensor_with_metadata_but_no_samples_warns(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n4\n") + (tmp_path / "EDA.csv").write_text("5680288411\n4\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -125,7 +133,7 @@ def test_e4_sensor_with_metadata_but_no_samples_warns(tmp_path): def test_e4_tags_first_row_is_checked_as_data(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411,unexpected\n1673588422\n") + (tmp_path / "tags.csv").write_text("5680288411,unexpected\n5680288422\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -151,7 +159,7 @@ def test_other_empty_files_and_phone_numbers_still_warn(tmp_path): def test_non_timestamp_phone_in_e4_tags_still_warns(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411.000000\n555-123-4567\n") + (tmp_path / "tags.csv").write_text("5680288411.000000\n555-123-4567\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -162,7 +170,7 @@ def test_non_timestamp_phone_in_e4_tags_still_warns(tmp_path): @pytest.mark.parametrize("sample_large_files", [True, False]) def test_e4_acc_quality_checks_each_data_channel(tmp_path, sample_large_files): (tmp_path / "ACC.csv").write_text( - "1673588411,1673588411,1673588411\n" + "5680288411,5680288411,5680288411\n" "32,32,32\n" ",,50\n,,51\n" ) @@ -177,7 +185,7 @@ def test_e4_acc_quality_checks_each_data_channel(tmp_path, sample_large_files): def test_e4_quality_range_excludes_timestamp_and_sample_rate(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n4\n0.5\n1.5\n") + (tmp_path / "EDA.csv").write_text("5680288411\n4\n0.5\n1.5\n") result = check_quality(tmp_path, ValidationConfig(value_ranges={"eda": (0, 1)})) @@ -211,7 +219,7 @@ def test_generic_csv_named_like_e4_sensor_keeps_its_header(tmp_path): def test_empty_tags_without_valid_e4_neighbor_remains_an_error(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n5\n0.123456\n") + (tmp_path / "EDA.csv").write_text("5680288411\n5\n0.123456\n") (tmp_path / "tags.csv").write_text("") filesystem = check_filesystem(tmp_path, ValidationConfig()) @@ -227,9 +235,9 @@ def test_empty_tags_without_valid_e4_neighbor_remains_an_error(tmp_path): def test_malformed_e4_tags_extra_fields_are_scanned_for_phi(tmp_path, sample_large_files): write_e4_sensor(tmp_path) (tmp_path / "tags.csv").write_text( - "1673588411.000000,person@example.com,555-123-4567\n" - "1673588422.000000\n" - "1673588433.000000,123-45-6789\n" + "5680288411.000000,person@example.com,555-123-4567\n" + "5680288422.000000\n" + "5680288433.000000,123-45-6789\n" ) result = check_privacy(tmp_path, ValidationConfig(sample_large_files=sample_large_files)) @@ -293,7 +301,7 @@ def test_synthetic_e4_example_validates_through_python_and_cli(tmp_path): def test_e4_hr_file_is_recognized_and_range_checked(tmp_path): - (tmp_path / "HR.csv").write_text("1673588421.000000\n1.000000\n72.00\n400.00\n") + (tmp_path / "HR.csv").write_text("5680288421.000000\n1.000000\n72.00\n400.00\n") assert is_e4_sensor_file(tmp_path / "HR.csv") result = check_quality(tmp_path, ValidationConfig(value_ranges={"hr": (20, 250)})) @@ -304,7 +312,7 @@ def test_e4_hr_file_is_recognized_and_range_checked(tmp_path): def test_e4_hr_confirms_tags_without_other_sensors(tmp_path): - (tmp_path / "HR.csv").write_text("1673588421.000000\n1.000000\n72.00\n") + (tmp_path / "HR.csv").write_text("5680288421.000000\n1.000000\n72.00\n") (tmp_path / "tags.csv").write_text("") result = validate_dataset( @@ -317,7 +325,7 @@ def test_e4_hr_confirms_tags_without_other_sensors(tmp_path): def test_e4_ibi_file_uses_offset_and_interval_columns(tmp_path): (tmp_path / "IBI.csv").write_text( - "1673588411.000000, IBI\n12.343750,0.828125\n13.156250,0.812500\n15.656250,2.500000\n" + "5680288411.000000, IBI\n12.343750,0.828125\n13.156250,0.812500\n15.656250,2.500000\n" ) assert is_e4_ibi_file(tmp_path / "IBI.csv") @@ -335,7 +343,7 @@ def test_e4_ibi_file_uses_offset_and_interval_columns(tmp_path): def test_e4_ibi_with_no_beats_is_information_not_a_warning(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -344,7 +352,7 @@ def test_e4_ibi_with_no_beats_is_information_not_a_warning(tmp_path): def test_e4_ibi_still_reports_malformed_rows(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n12.343750\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n12.343750\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -359,8 +367,8 @@ def test_generic_csv_named_ibi_keeps_its_header(tmp_path): def test_lowercase_e4_file_names_are_recognized(tmp_path): - (tmp_path / "eda.csv").write_text("1673588411\n4\n0.123456\n") - (tmp_path / "acc.csv").write_text("1673588411, 1673588411, 1673588411\n32, 32, 32\n-1, 63, 12\n") + (tmp_path / "eda.csv").write_text("5680288411\n4\n0.123456\n") + (tmp_path / "acc.csv").write_text("5680288411, 5680288411, 5680288411\n32, 32, 32\n-1, 63, 12\n") (tmp_path / "TAGS.csv").write_text("") result = validate_dataset( @@ -373,8 +381,8 @@ def test_lowercase_e4_file_names_are_recognized(tmp_path): def test_upper_case_csv_and_txt_extensions_are_validated(tmp_path): - (tmp_path / "EDA.CSV").write_text("1673588411\n4\n0.123456\n0.234567,0.1\n") - (tmp_path / "TAGS.CSV").write_text("1673588411\n555-123-4567\n") + (tmp_path / "EDA.CSV").write_text("5680288411\n4\n0.123456\n0.234567,0.1\n") + (tmp_path / "TAGS.CSV").write_text("5680288411\n555-123-4567\n") (tmp_path / "contacts.Csv").write_text("contact\nperson@example.com\n") (tmp_path / "NOTES.TXT").write_text("Call 555-987-6543\n") @@ -389,7 +397,7 @@ def test_upper_case_csv_and_txt_extensions_are_validated(tmp_path): def test_upper_case_csv_extension_is_quality_checked(tmp_path): - (tmp_path / "TEMP.CSV").write_text("1673588411\n4\n31.5\n99.0\n") + (tmp_path / "TEMP.CSV").write_text("5680288411\n4\n31.5\n99.0\n") result = check_quality(tmp_path, ValidationConfig(value_ranges={"temp": (30, 40)})) @@ -398,7 +406,7 @@ def test_upper_case_csv_extension_is_quality_checked(tmp_path): def test_phone_number_in_e4_tags_extra_field_is_reported(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411,2125551234\n") + (tmp_path / "tags.csv").write_text("5680288411,2125551234\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -408,7 +416,7 @@ def test_phone_number_in_e4_tags_extra_field_is_reported(tmp_path): def test_ibi_file_confirms_empty_e4_tags(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n12.343750,0.828125\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n12.343750,0.828125\n") (tmp_path / "tags.csv").write_text("") result = validate_dataset(