From fb5d18beacf8a5de884980561d62c18eb395553c Mon Sep 17 00:00:00 2001 From: Tom Pollard Date: Tue, 20 Oct 2026 09:22:09 -0400 Subject: [PATCH 1/2] Warn about dates that may be real in the privacy check The README and check_privacy docstring said dates were checked, but no date check existed. Add one: dates between date_lookback_years ago (200 by default) and today are reported as possibly real, since de-identified datasets usually shift dates. Future dates are treated as shifted. Written dates are found in CSV cells and text files. Unix seconds and milliseconds are only read in columns with time-like names, or in layout timestamp columns and metadata rows such as E4 start times, because IDs and counters have the same shape. Each file gets one warning listing its columns and date ranges; after 10 files the rest are summarized. E4 test fixtures now use timestamps shifted to 2150, so format tests stay independent of the date check. --- README.md | 14 +- physionet/validate/checks/csv_layout.py | 2 + physionet/validate/checks/dates.py | 158 +++++++++++++++++++++++ physionet/validate/checks/empatica.py | 9 +- physionet/validate/checks/privacy.py | 107 +++++++++++++++- physionet/validate/config.py | 3 + physionet/validate/models.py | 11 +- tests/fixtures/e4_session/ACC.csv | 2 +- tests/fixtures/e4_session/BVP.csv | 2 +- tests/fixtures/e4_session/EDA.csv | 2 +- tests/fixtures/e4_session/HR.csv | 2 +- tests/fixtures/e4_session/IBI.csv | 2 +- tests/fixtures/e4_session/README.md | 3 +- tests/fixtures/e4_session/TEMP.csv | 2 +- tests/fixtures/e4_session/tags.csv | 4 +- tests/validate/test_csv_layout.py | 4 +- tests/validate/test_dates.py | 164 ++++++++++++++++++++++++ tests/validate/test_empatica.py | 66 +++++----- 18 files changed, 503 insertions(+), 54 deletions(-) create mode 100644 physionet/validate/checks/dates.py create mode 100644 tests/validate/test_dates.py diff --git a/README.md b/README.md index db12729a..93905c19 100644 --- a/README.md +++ b/README.md @@ -139,12 +139,20 @@ dataset directory. - **documentation** - Missing or incomplete documentation (e.g. `README.md`). - **integrity** - CSV structure, encoding, and duplicate column detection. - **quality** - Missing values, outliers, and data type consistency. -- **privacy** - PHI patterns (SSN, email, phone numbers), date patterns, and - sensitive file detection. +- **privacy** - PHI patterns (SSN, email, phone numbers), ages above 89, + dates that may be real, and sensitive file detection. + +Dates from the last 200 years up to today are reported as possibly real, since +de-identified datasets usually shift dates (often into the future). This covers +written dates in CSV and text files, and Unix timestamps in columns with +time-like names such as `charttime` or `timestamp`. Dates shifted into that +window still need to be confirmed by the submitter. Set +`ValidationConfig(date_lookback_years=...)` to change the window, or `None` to +turn the check off. CSV exports from the Empatica E4 wearable are recognized, so their headerless layout, empty `tags.csv` files, and Unix timestamps don't produce false errors -or privacy warnings. This doesn't check that timestamps are de-identified. +or phone-number warnings. Their timestamps are checked as dates. **Examples:** diff --git a/physionet/validate/checks/csv_layout.py b/physionet/validate/checks/csv_layout.py index af21c5b8..0a64047b 100644 --- a/physionet/validate/checks/csv_layout.py +++ b/physionet/validate/checks/csv_layout.py @@ -17,6 +17,8 @@ class CsvLayout: columns: Optional[Tuple[str, ...]] = None metadata_rows: int = 0 + # Optional names for the metadata rows, used in messages. + metadata_row_names: Tuple[str, ...] = () allow_empty: bool = False empty_message: Optional[str] = None no_data_message: Optional[str] = None diff --git a/physionet/validate/checks/dates.py b/physionet/validate/checks/dates.py new file mode 100644 index 00000000..bdbb40dd --- /dev/null +++ b/physionet/validate/checks/dates.py @@ -0,0 +1,158 @@ +"""Find dates and Unix timestamps that fall in a realistic, possibly real, range. + +De-identified datasets usually shift dates, often far into the future. A date +between `lookback_years` ago and today may be a real date, so the privacy check +asks the submitter to confirm it was shifted. Format alone cannot tell shifted +dates from real ones, so findings are warnings to review, not errors. + +Recognized forms: +- Year-first dates: 2023-01-13, 2023/01/13, 2023.01.13, with an optional time. +- Day- or month-first dates with a four-digit year: 13/01/2023, 01-13-2023. +- Month names: 13 Jan 2023, January 13, 2023, Jan 2023. +- Unix seconds (9-10 digits) or milliseconds (12-13 digits), but only in + columns whose name suggests a time (or that a layout names), because IDs and + counters have the same shape. + +Two-digit years, eight-digit compact dates (20230113), times with no date, and +years on their own are not reported. +""" + +import re +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from typing import Iterator, Optional + + +_MONTH_NAME = ( + r"(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?" + r"|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\.?" +) +_MONTHS = {name: number for number, name in enumerate( + ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], start=1 +)} + +_YEAR_FIRST = re.compile(r"(? Optional[date]: + try: + return date(year, month, day) + except ValueError: + return None + + +def _month(name: str) -> int: + return _MONTHS[name.lower()[:3]] + + +def dates_in_text(text: str) -> Iterator[date]: + """Yield every calendar date written in text.""" + if not _MAYBE_WRITTEN_DATE.search(text) or not any(char.isdigit() for char in text): + return + for match in _YEAR_FIRST.finditer(text): + found = _make_date(int(match[1]), int(match[3]), int(match[4])) + if found: + yield found + for match in _YEAR_LAST.finditer(text): + first, second, year = int(match[1]), int(match[3]), int(match[4]) + # Ambiguous order: accept month-first or day-first, whichever is valid. + found = _make_date(year, first, second) or _make_date(year, second, first) + if found: + yield found + if not any(char.isalpha() for char in text): + return + full_date_spans = [] + for match in _DAY_MONTH_YEAR.finditer(text): + found = _make_date(int(match[3]), _month(match[2]), int(match[1])) + if found: + full_date_spans.append(match.span()) + yield found + for match in _MONTH_DAY_YEAR.finditer(text): + found = _make_date(int(match[3]), _month(match[1]), int(match[2])) + if found: + full_date_spans.append(match.span()) + yield found + for match in _MONTH_YEAR.finditer(text): + # "5 May 2019" also contains "May 2019"; count only the full date. + if any(start < match.end() and match.start() < end for start, end in full_date_spans): + continue + found = _make_date(int(match[2]), _month(match[1]), 1) + if found: + yield found + + +def unix_timestamp_date(value: str) -> Optional[date]: + """Return the UTC date of Unix seconds or milliseconds, if value is one.""" + match = _UNIX.match(value.strip()) + if not match: + return None + digits = match[1] + seconds = int(digits) / 1000 if len(digits) > 10 else int(digits) + try: + return datetime.fromtimestamp(seconds, tz=timezone.utc).date() + except (OverflowError, OSError, ValueError): + return None + + +def is_time_like_column(name: Optional[str]) -> bool: + """Guess from a column name whether it holds times (e.g. charttime, ts, date).""" + return bool(name) and bool(_TIME_LIKE_COLUMN.search(name)) + + +@dataclass +class DateRange: + """Earliest and latest realistic dates seen in one place.""" + + earliest: date + latest: date + + def add(self, found: date) -> None: + self.earliest = min(self.earliest, found) + self.latest = max(self.latest, found) + + def describe(self) -> str: + if self.earliest == self.latest: + return self.earliest.isoformat() + return f"{self.earliest.isoformat()} to {self.latest.isoformat()}" + + +class RealisticDateFinder: + """Collect dates between lookback_years ago and today.""" + + def __init__(self, lookback_years: int, today: Optional[date] = None): + today = today or datetime.now(timezone.utc).date() + self.lookback_years = lookback_years + self.start = date(today.year - lookback_years, 1, 1) + self.end = today + timedelta(days=1) # Allow for time-zone differences. + + def in_range(self, found: date) -> bool: + return self.start <= found <= self.end + + def scan(self, value: str, unix_allowed: bool, ranges: dict, key: str) -> None: + """Record realistic dates in value under key in ranges.""" + candidates = list(dates_in_text(value)) + if unix_allowed: + unix_date = unix_timestamp_date(value) + if unix_date: + candidates.append(unix_date) + for found in candidates: + if not self.in_range(found): + continue + if key in ranges: + ranges[key].add(found) + else: + ranges[key] = DateRange(found, found) diff --git a/physionet/validate/checks/empatica.py b/physionet/validate/checks/empatica.py index 64c9b8d6..1a0fa1ec 100644 --- a/physionet/validate/checks/empatica.py +++ b/physionet/validate/checks/empatica.py @@ -11,6 +11,8 @@ treated as E4 when a recognized sensor or IBI file sits beside it, and only its event_time column is exempt from the phone-number pattern. +Start times and event times are still checked as dates by the privacy check. + File names are matched in any letter case. tests/fixtures/e4_session holds a small synthetic example. """ @@ -101,11 +103,16 @@ def get_e4_csv_layout(file_path: Path) -> Optional[CsvLayout]: """Describe recognized E4 sensor, IBI, and event-tag files for CSV checks.""" columns = e4_sensor_columns(file_path) if columns is not None: - return CsvLayout(columns=columns, metadata_rows=2) + return CsvLayout( + columns=columns, + metadata_rows=2, + metadata_row_names=("start_time", "sample_rate"), + ) if is_e4_ibi_file(file_path): return CsvLayout( columns=_IBI_COLUMNS, metadata_rows=1, + metadata_row_names=("start_time",), no_data_message="No Empatica E4 inter-beat intervals detected (IBI.csv has only its start-time row)", ) if is_e4_tags_file(file_path): diff --git a/physionet/validate/checks/privacy.py b/physionet/validate/checks/privacy.py index 1eb530f0..0635011c 100644 --- a/physionet/validate/checks/privacy.py +++ b/physionet/validate/checks/privacy.py @@ -1,14 +1,21 @@ """Privacy and PHI validation checks.""" +import csv import os import re from pathlib import Path -from typing import Optional, Callable +from typing import Callable, Dict, List, Optional, Tuple from physionet.validate.models import CheckResult, ValidationIssue, CheckCategory, Severity from physionet.validate.config import ValidationConfig from physionet.validate.checks.csv_formats import find_files_by_suffix, get_csv_layout from physionet.validate.checks.csv_layout import is_unix_timestamp +from physionet.validate.checks.dates import ( + DateRange, RealisticDateFinder, dates_in_text, is_time_like_column, +) + +# Report realistic dates for at most this many files, then summarize the rest. +MAX_DATE_WARNING_FILES = 10 # Pattern names for better error messages PHI_PATTERN_NAMES = { @@ -58,7 +65,7 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio - PHI pattern detection - Age de-identification - Sensitive configuration files (keys, credentials) - - Date patterns + - Dates within config.date_lookback_years of today, which may be real Args: path: Path to dataset directory @@ -79,6 +86,11 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio pattern_info = [(re.compile(pattern), PHI_PATTERN_NAMES.get(pattern, "unknown pattern")) for pattern in config.phi_patterns] + date_finder = ( + RealisticDateFinder(config.date_lookback_years) if config.date_lookback_years else None + ) + date_findings: List[Tuple[str, Dict[Optional[str], DateRange]]] = [] + # Check CSV files csv_files = find_files_by_suffix(path, ".csv") for i, csv_file in enumerate(csv_files): @@ -88,7 +100,7 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio if any(p in str(csv_file) for p in config.ignore_patterns): continue - _check_csv_privacy(csv_file, path, config, pattern_info, result) + _check_csv_privacy(csv_file, path, config, pattern_info, result, date_finder, date_findings) # Check text files for PHI text_files = find_files_by_suffix(path, ".txt") @@ -99,11 +111,57 @@ def check_privacy(path: Path, config: ValidationConfig, progress_callback: Optio if any(p in str(text_file) for p in config.ignore_patterns): continue - _check_text_file_privacy(text_file, path, pattern_info, result, config) + _check_text_file_privacy( + text_file, path, pattern_info, result, config, date_finder, date_findings + ) + + if date_finder: + _report_realistic_dates(date_findings, config.date_lookback_years, result) return result +def _report_realistic_dates( + findings: List[Tuple[str, Dict[Optional[str], DateRange]]], + lookback_years: int, + result: CheckResult, +) -> None: + """Warn about files holding dates that may be real, one warning per file.""" + for relative_path, ranges in findings[:MAX_DATE_WARNING_FILES]: + places = [] + for column, date_range in ranges.items(): + where = f"column '{column}': " if column is not None else "" + places.append(f"{where}{date_range.describe()}") + columns = [column for column in ranges if column is not None] + result.issues.append( + ValidationIssue( + severity=Severity.WARNING, + category=CheckCategory.PRIVACY, + file=relative_path, + column=columns[0] if len(columns) == 1 else None, + message=( + f"Dates within the last {lookback_years} years may be real " + f"({'; '.join(places)})" + ), + suggestion="Confirm these dates have been shifted or removed", + ) + ) + remaining = len(findings) - MAX_DATE_WARNING_FILES + if remaining > 0: + result.issues.append( + ValidationIssue( + severity=Severity.WARNING, + category=CheckCategory.PRIVACY, + message=( + f"Dates within the last {lookback_years} years also found in " + f"{remaining} more file{'s' if remaining != 1 else ''} " + f"(showing first {MAX_DATE_WARNING_FILES})" + ), + suggestion="Confirm these dates have been shifted or removed", + ) + ) + + def _check_sensitive_files(path: Path, config: ValidationConfig, result: CheckResult) -> None: """Check for sensitive configuration files that shouldn't be in the dataset.""" for root, dirs, files in os.walk(path): @@ -167,10 +225,13 @@ def _check_csv_privacy( base_path: Path, config: ValidationConfig, pattern_info: list, - result: CheckResult + result: CheckResult, + date_finder: Optional[RealisticDateFinder] = None, + date_findings: Optional[list] = None, ) -> None: """Check a CSV file for privacy issues.""" relative_path = str(csv_file.relative_to(base_path)) + date_ranges: Dict[Optional[str], DateRange] = {} # Report each pattern at most once per column, including malformed extra fields. phi_columns = {} # {column: set of pattern names} @@ -178,6 +239,15 @@ def _check_csv_privacy( layout = get_csv_layout(csv_file) try: + if date_finder and layout.columns is not None and layout.metadata_rows: + # Headerless layouts may keep start times in rows before the data. + with open(csv_file, "r", encoding="utf-8", newline="") as f: + for index, row in zip(range(layout.metadata_rows), csv.reader(f)): + names = layout.metadata_row_names + label = names[index] if index < len(names) else f"metadata row {index + 1}" + for value in row: + date_finder.scan(value, True, date_ranges, label) + with open(csv_file, "r", encoding="utf-8") as f: reader = layout.dict_reader(f) @@ -223,6 +293,13 @@ def _check_csv_privacy( value_str = str(value).strip() + if date_finder: + unix_allowed = raw_col is not None and ( + raw_col in layout.unix_timestamp_columns + or is_time_like_column(raw_col) + ) + date_finder.scan(value_str, unix_allowed, date_ranges, col) + for pattern, pattern_name in pattern_info: if (pattern_name == "phone number" and raw_col in layout.unix_timestamp_columns @@ -266,6 +343,9 @@ def _check_csv_privacy( ) ) + if date_ranges and date_findings is not None: + date_findings.append((relative_path, date_ranges)) + except Exception as e: result.issues.append( ValidationIssue( @@ -277,10 +357,19 @@ def _check_csv_privacy( ) -def _check_text_file_privacy(text_file: Path, base_path: Path, pattern_info: list, result: CheckResult, config: ValidationConfig) -> None: +def _check_text_file_privacy( + text_file: Path, + base_path: Path, + pattern_info: list, + result: CheckResult, + config: ValidationConfig, + date_finder: Optional[RealisticDateFinder] = None, + date_findings: Optional[list] = None, +) -> None: """Check a text file for privacy issues.""" relative_path = str(text_file.relative_to(base_path)) detected_patterns = set() + date_ranges: Dict[Optional[str], DateRange] = {} try: with open(text_file, "r", encoding="utf-8") as f: @@ -291,6 +380,12 @@ def _check_text_file_privacy(text_file: Path, base_path: Path, pattern_info: lis for pattern, pattern_name in pattern_info: if pattern.search(line): detected_patterns.add(pattern_name) + if date_finder: + # Free text has no column names, so Unix timestamps are not inferred. + date_finder.scan(line, False, date_ranges, None) + + if date_ranges and date_findings is not None: + date_findings.append((relative_path, date_ranges)) # Report once per file with specific patterns found if detected_patterns: diff --git a/physionet/validate/config.py b/physionet/validate/config.py index f596d689..c5f225b6 100644 --- a/physionet/validate/config.py +++ b/physionet/validate/config.py @@ -37,6 +37,9 @@ class ValidationConfig: # Privacy settings allowed_age_max: int = 89 + # Warn about dates from this many years ago up to today, which may be real + # rather than shifted. None turns the date check off. + date_lookback_years: Optional[int] = 200 phi_patterns: List[str] = field(default_factory=lambda: [ r"\b\d{3}-\d{2}-\d{4}\b", # SSN pattern r"\b[\w\.-]+@[\w\.-]+\.\w+\b", # Email pattern diff --git a/physionet/validate/models.py b/physionet/validate/models.py index 6fc105d0..ecd4cb66 100644 --- a/physionet/validate/models.py +++ b/physionet/validate/models.py @@ -186,10 +186,13 @@ def summary(self) -> str: for issue in result.issues: icon = "✗" if issue.severity == Severity.ERROR else "⚠" - location = f" {issue.file}" - if issue.line: - location += f":{issue.line}" - lines.append(f" {icon}{location} - {issue.message}") + if issue.file: + location = f" {issue.file}" + if issue.line: + location += f":{issue.line}" + lines.append(f" {icon}{location} - {issue.message}") + else: + lines.append(f" {icon} {issue.message}") lines.append("") diff --git a/tests/fixtures/e4_session/ACC.csv b/tests/fixtures/e4_session/ACC.csv index 3912e000..409d3e39 100644 --- a/tests/fixtures/e4_session/ACC.csv +++ b/tests/fixtures/e4_session/ACC.csv @@ -1,4 +1,4 @@ -1673588411.000000,1673588411.000000,1673588411.000000 +5680288411.000000,5680288411.000000,5680288411.000000 32.000000,32.000000,32.000000 27,-25,50 28,-24,49 diff --git a/tests/fixtures/e4_session/BVP.csv b/tests/fixtures/e4_session/BVP.csv index 5871955e..9422c9b4 100644 --- a/tests/fixtures/e4_session/BVP.csv +++ b/tests/fixtures/e4_session/BVP.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 64.000000 0.123456 0.234567 diff --git a/tests/fixtures/e4_session/EDA.csv b/tests/fixtures/e4_session/EDA.csv index d2eaa83e..e2a9fdf3 100644 --- a/tests/fixtures/e4_session/EDA.csv +++ b/tests/fixtures/e4_session/EDA.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 4.000000 0.123456 0.234567 diff --git a/tests/fixtures/e4_session/HR.csv b/tests/fixtures/e4_session/HR.csv index cc04ba8a..aad17f98 100644 --- a/tests/fixtures/e4_session/HR.csv +++ b/tests/fixtures/e4_session/HR.csv @@ -1,4 +1,4 @@ -1673588421.000000 +5680288421.000000 1.000000 72.00 72.50 diff --git a/tests/fixtures/e4_session/IBI.csv b/tests/fixtures/e4_session/IBI.csv index aa71cc3d..4cdb509d 100644 --- a/tests/fixtures/e4_session/IBI.csv +++ b/tests/fixtures/e4_session/IBI.csv @@ -1,3 +1,3 @@ -1673588411.000000, IBI +5680288411.000000, IBI 12.343750,0.828125 13.156250,0.812500 diff --git a/tests/fixtures/e4_session/README.md b/tests/fixtures/e4_session/README.md index 396e5bf4..ffe479d6 100644 --- a/tests/fixtures/e4_session/README.md +++ b/tests/fixtures/e4_session/README.md @@ -4,4 +4,5 @@ This tiny, invented export is test data for the PhysioNet validator. It follows the E4 Connect CSV layout: a Unix start time, a sample rate, then signal samples. IBI.csv instead starts with the start time and `IBI`, followed by rows of seconds since the start and inter-beat interval in seconds. -The timestamps and signal values do not describe a real person or recording. +The timestamps are set in 2150, as a date-shifted dataset might be, and the +signal values do not describe a real person or recording. diff --git a/tests/fixtures/e4_session/TEMP.csv b/tests/fixtures/e4_session/TEMP.csv index 85cb5bca..22e5c697 100644 --- a/tests/fixtures/e4_session/TEMP.csv +++ b/tests/fixtures/e4_session/TEMP.csv @@ -1,4 +1,4 @@ -1673588411.000000 +5680288411.000000 4.000000 32.500000 32.600000 diff --git a/tests/fixtures/e4_session/tags.csv b/tests/fixtures/e4_session/tags.csv index 1a88b290..0e21b31b 100644 --- a/tests/fixtures/e4_session/tags.csv +++ b/tests/fixtures/e4_session/tags.csv @@ -1,2 +1,2 @@ -1673588411.000000 -1673588422.000000 +5680288411.000000 +5680288422.000000 diff --git a/tests/validate/test_csv_layout.py b/tests/validate/test_csv_layout.py index 0493cad1..8085314f 100644 --- a/tests/validate/test_csv_layout.py +++ b/tests/validate/test_csv_layout.py @@ -10,8 +10,8 @@ def test_custom_headerless_layout_is_used_by_all_csv_checks(tmp_path, monkeypatch): (tmp_path / "signals.csv").write_text( "device export\n" - "1673588411,0.1\n" - "1673588422,\n" + "5680288411,0.1\n" + "5680288422,\n" ) def detect(path): diff --git a/tests/validate/test_dates.py b/tests/validate/test_dates.py new file mode 100644 index 00000000..da742fb9 --- /dev/null +++ b/tests/validate/test_dates.py @@ -0,0 +1,164 @@ +"""Dates that may be real are flagged for review; shifted or ambiguous values are not.""" + +from datetime import date + +import pytest + +from physionet.validate import ValidationConfig, validate_dataset +from physionet.validate.checks import check_privacy +from physionet.validate.models import CheckCategory +from physionet.validate.checks.dates import ( + RealisticDateFinder, dates_in_text, is_time_like_column, unix_timestamp_date, +) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("2023-01-13", date(2023, 1, 13)), + ("2023-01-13T08:15:00", date(2023, 1, 13)), + ("2023-01-13 08:15", date(2023, 1, 13)), + ("2023/1/13", date(2023, 1, 13)), + ("13/01/2023", date(2023, 1, 13)), + ("01-13-2023", date(2023, 1, 13)), + ("Admitted 13 Jan 2023 overnight", date(2023, 1, 13)), + ("January 13th, 2023", date(2023, 1, 13)), + ("Sept 2019", date(2019, 9, 1)), + ], +) +def test_written_dates_are_found(text, expected): + assert expected in list(dates_in_text(text)) + + +def test_full_date_is_not_also_counted_as_month_and_year(): + assert list(dates_in_text("seen 5 May 2019 and in June 2020")) == [ + date(2019, 5, 5), date(2020, 6, 1) + ] + + +@pytest.mark.parametrize( + "text", + [ + "0.123456", "13/01/23", "20230113", "08:15:00", "2023", "1.2.3", + "2023-13-45", "maybe 2023", "version 1.10.2", "-25.50", + ], +) +def test_non_dates_are_ignored(text): + assert list(dates_in_text(text)) == [] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("1673588411", date(2023, 1, 13)), + ("1673588411.250000", date(2023, 1, 13)), + ("1673588411000", date(2023, 1, 13)), + ("631152000", date(1990, 1, 1)), + ("12345678", None), + ("0673588411", None), + ("2023-01-13", None), + ], +) +def test_unix_seconds_and_milliseconds(value, expected): + assert unix_timestamp_date(value) == expected + + +@pytest.mark.parametrize( + ("name", "expected"), + [ + ("charttime", True), ("event_time", True), ("Timestamp", True), ("ts", True), + ("start_ts", True), ("admit_date", True), ("epoch", True), + ("subject_id", False), ("status", False), ("patients", False), (None, False), + ], +) +def test_time_like_column_names(name, expected): + assert is_time_like_column(name) is expected + + +def test_range_runs_from_lookback_years_ago_to_today(): + finder = RealisticDateFinder(200, today=date(2026, 10, 20)) + + assert finder.in_range(date(1826, 1, 1)) + assert finder.in_range(date(2026, 10, 20)) + assert not finder.in_range(date(1825, 12, 31)) + assert not finder.in_range(date(2150, 1, 1)) + + +def _date_warnings(result): + return [issue for issue in result.issues if "Dates within the last" in issue.message] + + +def test_csv_dates_are_reported_once_per_file_with_their_range(tmp_path): + (tmp_path / "events.csv").write_text( + "subject_id,charttime,note\n" + "10000032,2019-03-04 10:00,seen 5 May 2019\n" + "10000032,2019-05-01 08:30,\n" + ) + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert len(warnings) == 1 + assert warnings[0].file == "events.csv" + assert "column 'charttime': 2019-03-04 to 2019-05-01" in warnings[0].message + assert "column 'note': 2019-05-05" in warnings[0].message + + +def test_shifted_future_dates_are_not_reported(tmp_path): + (tmp_path / "events.csv").write_text( + "charttime,epoch\n2180-07-23 10:00,6531840000\n" + ) + + assert _date_warnings(check_privacy(tmp_path, ValidationConfig())) == [] + + +def test_unix_timestamps_need_a_time_like_column(tmp_path): + (tmp_path / "ids.csv").write_text("record_id\n1673588411\n") + (tmp_path / "times.csv").write_text("timestamp_ms\n1673588411000\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [(issue.file, issue.column) for issue in warnings] == [("times.csv", "timestamp_ms")] + + +def test_text_file_dates_are_reported(tmp_path): + (tmp_path / "notes.txt").write_text("Patient admitted on 13 Jan 2023.\nNo other dates.\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [issue.file for issue in warnings] == ["notes.txt"] + assert "2023-01-13" in warnings[0].message + + +def test_e4_start_times_and_event_times_are_reported_as_dates_not_phones(tmp_path): + (tmp_path / "EDA.csv").write_text("1673588411.000000\n4.000000\n0.123456\n") + (tmp_path / "tags.csv").write_text("1673588500.000000\n") + + result = check_privacy(tmp_path, ValidationConfig()) + + assert not any("phone number" in issue.message for issue in result.issues) + messages = {issue.file: issue.message for issue in _date_warnings(result)} + assert "column 'start_time': 2023-01-13" in messages["EDA.csv"] + assert "column 'event_time': 2023-01-13" in messages["tags.csv"] + + +def test_date_check_can_be_turned_off(tmp_path): + (tmp_path / "events.csv").write_text("charttime\n2019-03-04\n") + + result = check_privacy(tmp_path, ValidationConfig(date_lookback_years=None)) + + assert _date_warnings(result) == [] + + +def test_many_files_with_dates_are_summarized(tmp_path): + for number in range(13): + (tmp_path / f"visit_{number:02d}.csv").write_text("charttime\n2019-03-04\n") + + result = validate_dataset( + str(tmp_path), ValidationConfig(check_documentation=False), show_progress=False + ) + warnings = _date_warnings(result.check_results[CheckCategory.PRIVACY]) + + assert len(warnings) == 11 + assert warnings[-1].file is None + assert "3 more files (showing first 10)" in warnings[-1].message + assert " ⚠ Dates within the last 200 years also found in 3 more files" in result.summary() diff --git a/tests/validate/test_empatica.py b/tests/validate/test_empatica.py index 5d03c9e3..84c30555 100644 --- a/tests/validate/test_empatica.py +++ b/tests/validate/test_empatica.py @@ -11,7 +11,7 @@ from physionet.validate.checks import check_filesystem, check_integrity, check_privacy, check_quality from physionet.validate.checks.csv_layout import is_unix_timestamp from physionet.validate.checks.empatica import is_e4_ibi_file, is_e4_sensor_file -from physionet.validate.models import Severity +from physionet.validate.models import CheckCategory, Severity E4_EXAMPLE = Path(__file__).parent.parent / "fixtures" / "e4_session" @@ -19,13 +19,17 @@ def write_e4_sensor(directory): (directory / "EDA.csv").write_text( - "1673588411.000000\n4.000000\n0.123456\n0.234567\n0.345678\n" + "5680288411.000000\n4.000000\n0.123456\n0.234567\n0.345678\n" ) -@pytest.mark.parametrize("timestamp", ["1673588411", "631152000", "5680281600"]) +@pytest.mark.parametrize( + ("timestamp", "realistic"), + [("1673588411", True), ("631152000", True), ("5680281600", False)], +) @pytest.mark.parametrize("empty_tags", [True, False]) -def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, empty_tags): +def test_integer_and_shifted_e4_timestamps(tmp_path, timestamp, realistic, empty_tags): + """Format checks pass either way; only dates up to today get a date warning.""" (tmp_path / "ACC.csv").write_text( f"{timestamp},{timestamp},{timestamp}\n" "32,32,32\n" @@ -41,8 +45,12 @@ def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, ) assert result.total_errors == 0 - assert result.total_warnings == 0 assert result.total_info == (1 if empty_tags else 0) + warnings = result.check_results[CheckCategory.PRIVACY].issues + expected_files = {"ACC.csv", "EDA.csv"} | (set() if empty_tags else {"tags.csv"}) + assert result.total_warnings == len(warnings) + assert {issue.file for issue in warnings} == (expected_files if realistic else set()) + assert all("may be real" in issue.message for issue in warnings) @pytest.mark.parametrize( @@ -50,7 +58,7 @@ def test_integer_and_shifted_e4_timestamps_validate_cleanly(tmp_path, timestamp, [ ("631152000", True), ("5680281600.123456", True), - ("1673588411", True), + ("5680288411", True), ("63115200", False), ("56802816000", False), ("-631152000", False), @@ -66,7 +74,7 @@ def test_unix_timestamp_shape(timestamp, expected): def test_e4_headerless_sensor_files_do_not_trigger_csv_structure_errors(tmp_path): write_e4_sensor(tmp_path) (tmp_path / "ACC.csv").write_text( - "1673588411.000000,1673588411.000000,1673588411.000000\n" + "5680288411.000000,5680288411.000000,5680288411.000000\n" "32.000000,32.000000,32.000000\n" "27,-25,50\n27,-24,50\n" ) @@ -79,7 +87,7 @@ def test_e4_headerless_sensor_files_do_not_trigger_csv_structure_errors(tmp_path def test_e4_tags_timestamps_are_not_reported_as_phone_numbers(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411.000000\n1673588422.000000\n") + (tmp_path / "tags.csv").write_text("5680288411.000000\n5680288422.000000\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -115,7 +123,7 @@ def test_e4_sensor_still_reports_malformed_rows(tmp_path): def test_e4_sensor_with_metadata_but_no_samples_warns(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n4\n") + (tmp_path / "EDA.csv").write_text("5680288411\n4\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -125,7 +133,7 @@ def test_e4_sensor_with_metadata_but_no_samples_warns(tmp_path): def test_e4_tags_first_row_is_checked_as_data(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411,unexpected\n1673588422\n") + (tmp_path / "tags.csv").write_text("5680288411,unexpected\n5680288422\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -151,7 +159,7 @@ def test_other_empty_files_and_phone_numbers_still_warn(tmp_path): def test_non_timestamp_phone_in_e4_tags_still_warns(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411.000000\n555-123-4567\n") + (tmp_path / "tags.csv").write_text("5680288411.000000\n555-123-4567\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -162,7 +170,7 @@ def test_non_timestamp_phone_in_e4_tags_still_warns(tmp_path): @pytest.mark.parametrize("sample_large_files", [True, False]) def test_e4_acc_quality_checks_each_data_channel(tmp_path, sample_large_files): (tmp_path / "ACC.csv").write_text( - "1673588411,1673588411,1673588411\n" + "5680288411,5680288411,5680288411\n" "32,32,32\n" ",,50\n,,51\n" ) @@ -177,7 +185,7 @@ def test_e4_acc_quality_checks_each_data_channel(tmp_path, sample_large_files): def test_e4_quality_range_excludes_timestamp_and_sample_rate(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n4\n0.5\n1.5\n") + (tmp_path / "EDA.csv").write_text("5680288411\n4\n0.5\n1.5\n") result = check_quality(tmp_path, ValidationConfig(value_ranges={"eda": (0, 1)})) @@ -211,7 +219,7 @@ def test_generic_csv_named_like_e4_sensor_keeps_its_header(tmp_path): def test_empty_tags_without_valid_e4_neighbor_remains_an_error(tmp_path): - (tmp_path / "EDA.csv").write_text("1673588411\n5\n0.123456\n") + (tmp_path / "EDA.csv").write_text("5680288411\n5\n0.123456\n") (tmp_path / "tags.csv").write_text("") filesystem = check_filesystem(tmp_path, ValidationConfig()) @@ -227,9 +235,9 @@ def test_empty_tags_without_valid_e4_neighbor_remains_an_error(tmp_path): def test_malformed_e4_tags_extra_fields_are_scanned_for_phi(tmp_path, sample_large_files): write_e4_sensor(tmp_path) (tmp_path / "tags.csv").write_text( - "1673588411.000000,person@example.com,555-123-4567\n" - "1673588422.000000\n" - "1673588433.000000,123-45-6789\n" + "5680288411.000000,person@example.com,555-123-4567\n" + "5680288422.000000\n" + "5680288433.000000,123-45-6789\n" ) result = check_privacy(tmp_path, ValidationConfig(sample_large_files=sample_large_files)) @@ -293,7 +301,7 @@ def test_synthetic_e4_example_validates_through_python_and_cli(tmp_path): def test_e4_hr_file_is_recognized_and_range_checked(tmp_path): - (tmp_path / "HR.csv").write_text("1673588421.000000\n1.000000\n72.00\n400.00\n") + (tmp_path / "HR.csv").write_text("5680288421.000000\n1.000000\n72.00\n400.00\n") assert is_e4_sensor_file(tmp_path / "HR.csv") result = check_quality(tmp_path, ValidationConfig(value_ranges={"hr": (20, 250)})) @@ -304,7 +312,7 @@ def test_e4_hr_file_is_recognized_and_range_checked(tmp_path): def test_e4_hr_confirms_tags_without_other_sensors(tmp_path): - (tmp_path / "HR.csv").write_text("1673588421.000000\n1.000000\n72.00\n") + (tmp_path / "HR.csv").write_text("5680288421.000000\n1.000000\n72.00\n") (tmp_path / "tags.csv").write_text("") result = validate_dataset( @@ -317,7 +325,7 @@ def test_e4_hr_confirms_tags_without_other_sensors(tmp_path): def test_e4_ibi_file_uses_offset_and_interval_columns(tmp_path): (tmp_path / "IBI.csv").write_text( - "1673588411.000000, IBI\n12.343750,0.828125\n13.156250,0.812500\n15.656250,2.500000\n" + "5680288411.000000, IBI\n12.343750,0.828125\n13.156250,0.812500\n15.656250,2.500000\n" ) assert is_e4_ibi_file(tmp_path / "IBI.csv") @@ -335,7 +343,7 @@ def test_e4_ibi_file_uses_offset_and_interval_columns(tmp_path): def test_e4_ibi_with_no_beats_is_information_not_a_warning(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -344,7 +352,7 @@ def test_e4_ibi_with_no_beats_is_information_not_a_warning(tmp_path): def test_e4_ibi_still_reports_malformed_rows(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n12.343750\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n12.343750\n") result = check_integrity(tmp_path, ValidationConfig()) @@ -359,8 +367,8 @@ def test_generic_csv_named_ibi_keeps_its_header(tmp_path): def test_lowercase_e4_file_names_are_recognized(tmp_path): - (tmp_path / "eda.csv").write_text("1673588411\n4\n0.123456\n") - (tmp_path / "acc.csv").write_text("1673588411, 1673588411, 1673588411\n32, 32, 32\n-1, 63, 12\n") + (tmp_path / "eda.csv").write_text("5680288411\n4\n0.123456\n") + (tmp_path / "acc.csv").write_text("5680288411, 5680288411, 5680288411\n32, 32, 32\n-1, 63, 12\n") (tmp_path / "TAGS.csv").write_text("") result = validate_dataset( @@ -373,8 +381,8 @@ def test_lowercase_e4_file_names_are_recognized(tmp_path): def test_upper_case_csv_and_txt_extensions_are_validated(tmp_path): - (tmp_path / "EDA.CSV").write_text("1673588411\n4\n0.123456\n0.234567,0.1\n") - (tmp_path / "TAGS.CSV").write_text("1673588411\n555-123-4567\n") + (tmp_path / "EDA.CSV").write_text("5680288411\n4\n0.123456\n0.234567,0.1\n") + (tmp_path / "TAGS.CSV").write_text("5680288411\n555-123-4567\n") (tmp_path / "contacts.Csv").write_text("contact\nperson@example.com\n") (tmp_path / "NOTES.TXT").write_text("Call 555-987-6543\n") @@ -389,7 +397,7 @@ def test_upper_case_csv_and_txt_extensions_are_validated(tmp_path): def test_upper_case_csv_extension_is_quality_checked(tmp_path): - (tmp_path / "TEMP.CSV").write_text("1673588411\n4\n31.5\n99.0\n") + (tmp_path / "TEMP.CSV").write_text("5680288411\n4\n31.5\n99.0\n") result = check_quality(tmp_path, ValidationConfig(value_ranges={"temp": (30, 40)})) @@ -398,7 +406,7 @@ def test_upper_case_csv_extension_is_quality_checked(tmp_path): def test_phone_number_in_e4_tags_extra_field_is_reported(tmp_path): write_e4_sensor(tmp_path) - (tmp_path / "tags.csv").write_text("1673588411,2125551234\n") + (tmp_path / "tags.csv").write_text("5680288411,2125551234\n") result = check_privacy(tmp_path, ValidationConfig()) @@ -408,7 +416,7 @@ def test_phone_number_in_e4_tags_extra_field_is_reported(tmp_path): def test_ibi_file_confirms_empty_e4_tags(tmp_path): - (tmp_path / "IBI.csv").write_text("1673588411.000000, IBI\n12.343750,0.828125\n") + (tmp_path / "IBI.csv").write_text("5680288411.000000, IBI\n12.343750,0.828125\n") (tmp_path / "tags.csv").write_text("") result = validate_dataset( From 1426a1ed27aecf44fb859b7440282106c7472516 Mon Sep 17 00:00:00 2001 From: Tom Pollard Date: Tue, 20 Oct 2026 09:32:36 -0400 Subject: [PATCH 2/2] Catch sentence-ending dates, validate the date window, keep every finding A day- or month-first date followed by a full stop ("13/01/2023.") was missed. A trailing period is now allowed; a decimal extension still is not. date_lookback_years must be a whole number from 1 to 1000 or None. Negative values silently matched nothing, 0 turned the check off, and very large values raised an error partway through validation. Every file with possible real dates now keeps its own warning, tagged with the code "possible-real-date", so JSON and Markdown reports list them all. Only the terminal output is shortened, to 10 warnings of one kind per category plus a count of the rest. --- README.md | 5 ++- physionet/cli.py | 6 ++- physionet/validate/checks/dates.py | 6 ++- physionet/validate/checks/privacy.py | 24 +++-------- physionet/validate/config.py | 15 +++++++ physionet/validate/models.py | 24 ++++++++++- tests/validate/test_dates.py | 59 +++++++++++++++++++++++++--- 7 files changed, 110 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 93905c19..18c3cbca 100644 --- a/README.md +++ b/README.md @@ -147,8 +147,9 @@ de-identified datasets usually shift dates (often into the future). This covers written dates in CSV and text files, and Unix timestamps in columns with time-like names such as `charttime` or `timestamp`. Dates shifted into that window still need to be confirmed by the submitter. Set -`ValidationConfig(date_lookback_years=...)` to change the window, or `None` to -turn the check off. +`ValidationConfig(date_lookback_years=...)` to a number of years from 1 to 1000 +to change the window, or `None` to turn the check off. The terminal shows date +warnings for the first 10 files; the saved report lists every file. CSV exports from the Empatica E4 wearable are recognized, so their headerless layout, empty `tags.csv` files, and Unix timestamps don't produce false errors diff --git a/physionet/cli.py b/physionet/cli.py index da2d64c3..afb51d2e 100644 --- a/physionet/cli.py +++ b/physionet/cli.py @@ -8,6 +8,9 @@ from physionet.validate import validate_dataset, ValidationConfig +# Issues of one kind (such as date warnings) shown per category in the terminal. +TERMINAL_MAX_PER_CODE = 10 + def main(): """Main entry point for the CLI.""" @@ -263,7 +266,8 @@ def _handle_validate(args): result = validate_dataset(str(dataset_path), config, show_progress=True) print() - print(result.summary()) + # Keep the terminal readable; the saved report below lists every issue. + print(result.summary(max_per_code=TERMINAL_MAX_PER_CODE)) # Save validation report - either to specified path or default location if args.report: diff --git a/physionet/validate/checks/dates.py b/physionet/validate/checks/dates.py index bdbb40dd..f7566784 100644 --- a/physionet/validate/checks/dates.py +++ b/physionet/validate/checks/dates.py @@ -22,6 +22,8 @@ from datetime import date, datetime, timedelta, timezone from typing import Iterator, Optional +from physionet.validate.config import validate_lookback_years + _MONTH_NAME = ( r"(jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?" @@ -32,7 +34,8 @@ )} _YEAR_FIRST = re.compile(r"(? None: """Warn about files holding dates that may be real, one warning per file.""" - for relative_path, ranges in findings[:MAX_DATE_WARNING_FILES]: + for relative_path, ranges in findings: places = [] for column, date_range in ranges.items(): where = f"column '{column}': " if column is not None else "" @@ -144,20 +145,7 @@ def _report_realistic_dates( f"({'; '.join(places)})" ), suggestion="Confirm these dates have been shifted or removed", - ) - ) - remaining = len(findings) - MAX_DATE_WARNING_FILES - if remaining > 0: - result.issues.append( - ValidationIssue( - severity=Severity.WARNING, - category=CheckCategory.PRIVACY, - message=( - f"Dates within the last {lookback_years} years also found in " - f"{remaining} more file{'s' if remaining != 1 else ''} " - f"(showing first {MAX_DATE_WARNING_FILES})" - ), - suggestion="Confirm these dates have been shifted or removed", + code=POSSIBLE_REAL_DATE, ) ) diff --git a/physionet/validate/config.py b/physionet/validate/config.py index c5f225b6..d286e67d 100644 --- a/physionet/validate/config.py +++ b/physionet/validate/config.py @@ -3,6 +3,17 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple +MAX_LOOKBACK_YEARS = 1000 + + +def validate_lookback_years(value) -> None: + """Reject date windows that would contain no dates or cannot be represented.""" + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= MAX_LOOKBACK_YEARS: + raise ValueError( + f"date_lookback_years must be a whole number from 1 to {MAX_LOOKBACK_YEARS}, " + f"or None to turn the date check off (got {value!r})" + ) + @dataclass class ValidationConfig: @@ -45,3 +56,7 @@ class ValidationConfig: r"\b[\w\.-]+@[\w\.-]+\.\w+\b", # Email pattern r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", # Phone pattern ]) + + def __post_init__(self) -> None: + if self.date_lookback_years is not None: + validate_lookback_years(self.date_lookback_years) diff --git a/physionet/validate/models.py b/physionet/validate/models.py index ecd4cb66..c30bae4f 100644 --- a/physionet/validate/models.py +++ b/physionet/validate/models.py @@ -34,6 +34,8 @@ class ValidationIssue: column: Optional[str] = None value: Optional[str] = None suggestion: Optional[str] = None + # Groups issues of one kind, e.g. so a summary can shorten long runs of them. + code: Optional[str] = None def to_dict(self) -> Dict[str, Any]: """Convert issue to dictionary format.""" @@ -52,6 +54,8 @@ def to_dict(self) -> Dict[str, Any]: result["value"] = self.value if self.suggestion: result["suggestion"] = self.suggestion + if self.code: + result["code"] = self.code return result @@ -126,8 +130,12 @@ def status(self) -> str: return "warning" return "pass" - def summary(self) -> str: - """Generate a human-readable summary.""" + def summary(self, max_per_code: Optional[int] = None) -> str: + """Generate a human-readable summary. + + With max_per_code, show at most that many issues sharing a code in each + category and count the rest. Issues without a code are always shown. + """ # Format timestamp as human-readable try: dt = datetime.fromisoformat(self.timestamp.replace('Z', '+00:00')) @@ -184,7 +192,14 @@ def summary(self) -> str: lines.append(f"{status_icon} {category.value.replace('_', ' ').title()}{issue_summary}") + shown_per_code: Dict[str, int] = {} + hidden_per_code: Dict[str, int] = {} for issue in result.issues: + if issue.code and max_per_code is not None: + if shown_per_code.get(issue.code, 0) >= max_per_code: + hidden_per_code[issue.code] = hidden_per_code.get(issue.code, 0) + 1 + continue + shown_per_code[issue.code] = shown_per_code.get(issue.code, 0) + 1 icon = "✗" if issue.severity == Severity.ERROR else "⚠" if issue.file: location = f" {issue.file}" @@ -193,6 +208,11 @@ def summary(self) -> str: lines.append(f" {icon}{location} - {issue.message}") else: lines.append(f" {icon} {issue.message}") + for code, hidden in hidden_per_code.items(): + lines.append( + f" … {hidden} more {code} issue{'s' if hidden != 1 else ''} " + f"not shown here; the saved report lists them all" + ) lines.append("") diff --git a/tests/validate/test_dates.py b/tests/validate/test_dates.py index da742fb9..8fb7fc95 100644 --- a/tests/validate/test_dates.py +++ b/tests/validate/test_dates.py @@ -1,5 +1,8 @@ """Dates that may be real are flagged for review; shifted or ambiguous values are not.""" +import json +import subprocess +import sys from datetime import date import pytest @@ -149,16 +152,62 @@ def test_date_check_can_be_turned_off(tmp_path): assert _date_warnings(result) == [] -def test_many_files_with_dates_are_summarized(tmp_path): +def test_every_file_with_dates_is_kept_and_only_the_terminal_is_shortened(tmp_path): for number in range(13): (tmp_path / f"visit_{number:02d}.csv").write_text("charttime\n2019-03-04\n") + (tmp_path / "README.md").write_text("Synthetic visits.\n") + report = tmp_path / "report.json" result = validate_dataset( str(tmp_path), ValidationConfig(check_documentation=False), show_progress=False ) warnings = _date_warnings(result.check_results[CheckCategory.PRIVACY]) + assert sorted(issue.file for issue in warnings) == [f"visit_{n:02d}.csv" for n in range(13)] + assert all(issue.to_dict()["code"] == "possible-real-date" for issue in warnings) - assert len(warnings) == 11 - assert warnings[-1].file is None - assert "3 more files (showing first 10)" in warnings[-1].message - assert " ⚠ Dates within the last 200 years also found in 3 more files" in result.summary() + assert result.summary().count("may be real") == 13 + short = result.summary(max_per_code=10) + assert short.count("may be real") == 10 + assert "3 more possible-real-date issues not shown here" in short + + cli = subprocess.run( + [sys.executable, "-m", "physionet", "validate", str(tmp_path), "--report", str(report)], + capture_output=True, text=True, + ) + assert cli.stdout.count("may be real") == 10 + assert "3 more possible-real-date issues" in cli.stdout + issues = json.loads(report.read_text())["checks"]["privacy"]["issues"] + assert len([issue for issue in issues if issue.get("code") == "possible-real-date"]) == 13 + + markdown = tmp_path / "PHYSIONET_REPORT.md" + subprocess.run([sys.executable, "-m", "physionet", "validate", str(tmp_path)], capture_output=True) + assert markdown.read_text().count("may be real") == 13 + + +@pytest.mark.parametrize( + "text", + ["Seen on 13/01/2023.", "Seen on 13/01/2023, then home.", "Seen on 13/01/2023"], +) +def test_sentence_ending_dates_are_reported(tmp_path, text): + (tmp_path / "notes.txt").write_text(text + "\n") + + warnings = _date_warnings(check_privacy(tmp_path, ValidationConfig())) + + assert [issue.file for issue in warnings] == ["notes.txt"] + assert "2023-01-13" in warnings[0].message + + +@pytest.mark.parametrize("text", ["13.01.2023.5", "1.13.01.2023"]) +def test_numbers_extending_a_date_are_not_dates(text): + assert list(dates_in_text(text)) == [] + + +@pytest.mark.parametrize("value", [-1, 0, 1001, 3000, 1.5, "200", True]) +def test_invalid_date_window_is_rejected(value): + with pytest.raises(ValueError, match="date_lookback_years"): + ValidationConfig(date_lookback_years=value) + + +@pytest.mark.parametrize("value", [None, 1, 200, 1000]) +def test_valid_date_windows_are_accepted(value): + assert ValidationConfig(date_lookback_years=value).date_lookback_years == value