Python Practice for Data Engineers

# Pattern 3: schema validation with structured errors.
# Interviewers reward error design over a bare ValueError because they've
# debugged silent data loss caused by swallowed exceptions.

from dataclasses import dataclass
from typing import Optional, Any

@dataclass
class FieldSpec:
    type: type
    required: bool = True
    max_len: Optional[int] = None
    choices: Optional[list[Any]] = None

@dataclass
class RowError:
    row_index: int
    field: str
    reason: str
    raw_value: Any

def validate_records(
    records: list[dict],
    schema: dict[str, FieldSpec],
) -> tuple[list[dict], list[RowError]]:
    clean, errors = [], []
    for i, record in enumerate(records):
        row_errors = []
        for field, spec in schema.items():
            value = record.get(field)
            if value is None:
                if spec.required:
                    row_errors.append(RowError(i, field, "required field missing", None))
                continue
            if not isinstance(value, spec.type):
                row_errors.append(RowError(i, field,
                    f"expected {spec.type.__name__}, got {type(value).__name__}", value))
                continue
            if spec.max_len is not None and hasattr(value, "__len__"):
                if len(value) > spec.max_len:
                    row_errors.append(RowError(i, field,
                        f"length {len(value)} > max {spec.max_len}", value))
            if spec.choices and value not in spec.choices:
                row_errors.append(RowError(i, field,
                    f"value not in allowed choices {spec.choices}", value))
        if row_errors:
            errors.extend(row_errors)
        else:
            clean.append(record)
    return clean, errors

# Why the structure:
#   row_index = where in the input batch
#   field     = which column to fix
#   reason    = human-readable
#   raw_value = for downstream replay / debugging
# All 4 are needed at 2am during an oncall.

Structured errors with row_index and field-level messages. The interview reward is in the error design, not just the happy path.