""" CMS ACCESS API — Data Reporting service. Builds US Core Observation resources from OAPMeasureObservation and submits $report-data (submit-only). The polling worker applies each final outcome via apply_reporting_result(), updating submission status on the observation record. """ import logging from datetime import datetime, timezone from typing import Any from fhirclient.models.codeableconcept import CodeableConcept from fhirclient.models.coding import Coding from fhirclient.models.fhirabstractbase import FHIRValidationError from fhirclient.models.fhirdatetime import FHIRDateTime from fhirclient.models.fhirreference import FHIRReference from fhirclient.models.observation import Observation from fhirclient.models.parameters import Parameters, ParametersParameter from fhirclient.models.quantity import Quantity from access_core.clinical.models import OAPMeasureObservation from access_core.cms_integration.models import CMSApiRequest from access_core.cms_integration.services import ( _is_terminal_error, note_failed_attempt, record_submission, ) from access_core.constants import DEFAULT_TRACK from access_core.services.cms_client import ( CmsApiError, CmsClient, describe_response_params, get_shared_cms_client, parse_fhir_resource, ) logger = logging.getLogger(__name__) class CmsReportingError(Exception): """Raised on data reporting failures.""" def _get_cms_client(): """Process-wide shared CMS client (token + session cached).""" return get_shared_cms_client() OAP_MEASURE_FHIR_MAPPING = { "blood_pressure_systolic": { "code": { "coding": [ { "system": "http://loinc.org", "code": "8480-6", "display": "Systolic blood pressure", } ] }, "unit": "mmHg", "ucum": "[mm_Hg]", }, "blood_pressure_diastolic": { "code": { "coding": [ { "system": "http://loinc.org", "code": "8462-4", "display": "Diastolic blood pressure", } ] }, "unit": "mmHg", "ucum": "[mm_Hg]", }, "ldl_cholesterol": { "code": { "coding": [ { "system": "http://loinc.org", "code": "2089-1", "display": "LDL Cholesterol", } ] }, "unit": "mg/dL", "ucum": "[mg/dL]", }, "total_cholesterol": { "code": { "coding": [ { "system": "http://loinc.org", "code": "2093-3", "display": "Total Cholesterol", } ] }, "unit": "mg/dL", "ucum": "[mg/dL]", }, "hba1c": { "code": { "coding": [ { "system": "http://loinc.org", "code": "4548-4", "display": "Hemoglobin A1c", } ] }, "unit": "%", "ucum": "[%]", }, "egfr": { "code": { "coding": [ { "system": "http://loinc.org", "code": "33914-3", "display": "Glomerular filtration rate", } ] }, "unit": "mL/min/1.73.m2", "ucum": "[mL]/min/[m2]", }, "uacr": { "code": { "coding": [ { "system": "http://loinc.org", "code": "30295-8", "display": "Urine albumin/creatinine", } ] }, "unit": "mg/g", "ucum": "[mg]/[g]", }, "weight": { "code": { "coding": [ { "system": "http://loinc.org", "code": "29463-7", "display": "Body weight", } ] }, "unit": "lbs", "ucum": "[lb_av]", }, "bmi": { "code": { "coding": [ { "system": "http://loinc.org", "code": "39156-5", "display": "Body mass index", } ] }, "unit": "kg/m2", "ucum": "[kg_m2]", }, } def build_reporting_parameters( observation: OAPMeasureObservation, ) -> Parameters: """ Build FHIR R4 Parameters resource for $report-data operation. Constructs a US Core Observation resource from the OAPMeasureObservation and maps it to the appropriate LOINC code based on measure_name. Args: observation: OAPMeasureObservation instance with approved data Returns: Typed fhirclient Parameters resource; serialize with .as_json() before POST. """ beneficiary = observation.beneficiary care_period = observation.care_period track_code = ( care_period.clinical_track.track_name if hasattr(care_period, "clinical_track") and care_period.clinical_track else DEFAULT_TRACK ) measure_key = _resolve_measure_key(observation.measure_definition.measure_name) fhir_mapping = OAP_MEASURE_FHIR_MAPPING.get(measure_key, {}) canonical_obs = observation.canonical_observation obs_resource = Observation() obs_resource.status = "final" category_coding = Coding() category_coding.system = ( "http://terminology.hl7.org/CodeSystem/observation-category" ) category_coding.code = "laboratory" category_concept = CodeableConcept() category_concept.coding = [category_coding] obs_resource.category = [category_concept] subject_ref = FHIRReference() subject_ref.reference = f"Patient/{beneficiary.id}" obs_resource.subject = subject_ref obs_resource.effectiveDateTime = FHIRDateTime( observation.collection_date.isoformat() ) code_concept: CodeableConcept | None = None if fhir_mapping: source_coding = fhir_mapping["code"]["coding"][0] mapping_coding = Coding() mapping_coding.system = source_coding.get("system", "http://loinc.org") mapping_coding.code = source_coding.get("code", "") mapping_coding.display = source_coding.get("display", "") code_concept = CodeableConcept() code_concept.coding = [mapping_coding] obs_resource.code = code_concept obs_resource.unit = fhir_mapping.get("unit", "") if canonical_obs.value_numeric is not None: quantity = Quantity() quantity.value = float(canonical_obs.value_numeric) quantity.unit = canonical_obs.unit or fhir_mapping.get("unit", "") ucum = fhir_mapping.get("ucum") quantity.system = ucum or "http://unitsofmeasure.org" quantity.code = ucum.strip("[]") if ucum else "" obs_resource.valueQuantity = quantity elif canonical_obs.value_text: obs_resource.valueString = canonical_obs.value_text if canonical_obs.loinc_code and code_concept is not None: loinc_coding = Coding() loinc_coding.system = "http://loinc.org" loinc_coding.code = canonical_obs.loinc_code.loinc_number loinc_coding.display = canonical_obs.loinc_code.common_name or "" code_concept.coding.append(loinc_coding) performer_ref = FHIRReference() performer_ref.reference = "Organization/metaflow-access" obs_resource.performer = [performer_ref] encounter_ref = FHIRReference() encounter_ref.reference = f"Encounter/{care_period.id}" obs_resource.encounter = encounter_ref resource_param = ParametersParameter() resource_param.name = "_resource" resource_param.resource = obs_resource track_param = ParametersParameter() track_param.name = "track" track_param.valueString = track_code type_param = ParametersParameter() type_param.name = "reporting-type" type_param.valueString = observation.reporting_type params = Parameters() params.parameter = [resource_param, track_param, type_param] return params def report_oap_observations( observations: list[OAPMeasureObservation] | None = None, beneficiary_id=None, care_period_id=None, limit: int | None = None, ) -> dict[str, Any]: """ Submit OAP measure data to CMS ACCESS API via $report-data (submit-only). Each observation is POSTed once; the durable CMSApiSubmissionStatus row is picked up by cms_polling_worker, which applies the final outcome through apply_reporting_result(). The pending marker set on submission keeps a row out of subsequent batches. Args: observations: Specific OAPMeasureObservation instances to report (overrides filters) beneficiary_id: Filter by beneficiary UUID (used when observations is None) care_period_id: Filter by care period UUID (used with beneficiary_id) limit: Maximum number of observations to report (0 or None = no limit) Returns: Summary dict with success/failure counts and per-observation results. """ client = _get_cms_client() results = { "submitted": 0, "rejected": 0, "failed": 0, "skipped": 0, "details": [], } if observations: obs_qs = OAPMeasureObservation.objects.filter( id__in=[o.id for o in observations] ) else: qs_filter: dict[str, Any] = {"status": "approved"} if beneficiary_id: qs_filter["beneficiary_id"] = beneficiary_id if care_period_id: qs_filter["care_period_id"] = care_period_id obs_qs = OAPMeasureObservation.objects.filter(**qs_filter) approved_obs = obs_qs.exclude(cms_submission_response__isnull=False).exclude( status="submitted_to_cms" ) if limit and limit > 0: approved_obs = approved_obs[:limit] if not approved_obs.exists(): logger.info("No unreported approved observations found") return results for observation in approved_obs: # Atomic pre-POST claim: the marker is set before any call to CMS, so # a concurrent batch run loses the claim and skips instead of posting # a duplicate $report-data. if not _claim_observation(observation): logger.info( f"OAP observation {observation.id} already claimed by another " "run; skipping" ) results["skipped"] += 1 continue try: result = _report_single_observation(client, observation) results["details"].append(result) if result["status"] == "submitted": results["submitted"] += 1 elif result["status"] == "rejected": results["rejected"] += 1 except CmsReportingError as exc: # Permanent for this payload (FHIR build error, missing submission # id) — reject the row; retrying would fail identically. logger.error( f"Reporting failed for OAP observation {observation.id}: {exc}" ) _reject_observation(observation, str(exc)) results["failed"] += 1 results["details"].append(_report_detail(observation, str(exc))) except CmsApiError as exc: if _is_terminal_error(exc): logger.error( f"Terminal CMS error (HTTP {exc.status_code}) reporting " f"OAP observation {observation.id}: {exc}" ) _reject_observation(observation, str(exc)) else: # Transient (5xx/429/network) — release the claim so the next # batch resubmits; this one row must not abort the whole run. logger.warning( f"Transient CMS error (HTTP {exc.status_code}) reporting " f"OAP observation {observation.id}; will retry: {exc}" ) _release_observation_claim(observation) results["failed"] += 1 results["details"].append(_report_detail(observation, str(exc))) except Exception as exc: # noqa: BLE001 - logged; row must not stay stuck claimed logger.error( f"Unexpected error reporting OAP observation {observation.id}: {exc}" ) _reject_observation(observation, f"unexpected: {exc}") results["failed"] += 1 results["details"].append(_report_detail(observation, str(exc))) logger.info( f"Data reporting batch complete: {results['submitted']} submitted, " f"{results['rejected']} rejected, {results['failed']} failed, " f"{results['skipped']} skipped out of {len(approved_obs)}" ) return results def reset_rejected_observations(queryset=None) -> int: """Reset rejected OAP observations so the next reporting batch resubmits them. Rejection (terminal CMS 4xx, FHIR build error, or a rejected $report-data outcome) leaves status="rejected" plus a non-null cms_submission_response; both keep the row out of report_oap_observations forever. Clearing them is the resubmission path — call after fixing whatever caused the rejection (e.g. corrected source data or a CMS-side issue). Returns rows requeued. """ if queryset is None: queryset = OAPMeasureObservation.objects.all() count = queryset.filter(status="rejected").update( status="approved", cms_submission_response=None ) logger.info(f"Requeued {count} rejected OAP observation(s) for CMS resubmission") return count def _reject_observation(observation: OAPMeasureObservation, error: str) -> None: observation.status = "rejected" observation.cms_submission_response = {"error": error} observation.save(update_fields=["status", "cms_submission_response"]) def _report_detail(observation: OAPMeasureObservation, error: str) -> dict[str, Any]: return { "observation_id": str(observation.id), "measure": observation.measure_definition.measure_name, "status": "error", "error": error, } def _claim_observation(observation: OAPMeasureObservation) -> bool: """Atomically mark an approved observation as pending CMS processing. The conditional UPDATE (marker still NULL) means exactly one concurrent batch run can claim a row, and the marker is set before any POST to CMS — so two overlapping runs can never submit the same observation twice. Returns False if another run already owns the row. """ return ( OAPMeasureObservation.objects.filter( pk=observation.pk, cms_submission_response__isnull=True ).update( cms_submission_response={ "status": "pending_cms_processing", "submission_id": None, } ) == 1 ) def _release_observation_claim(observation: OAPMeasureObservation) -> None: """Clear the pending marker so a transient failure retries next batch.""" OAPMeasureObservation.objects.filter(pk=observation.pk).update( cms_submission_response=None ) def _report_single_observation( client: CmsClient, observation: OAPMeasureObservation ) -> dict[str, Any]: """Submit a single OAP observation to CMS. Returns result detail dict.""" params = build_reporting_parameters(observation) try: payload = params.as_json() except FHIRValidationError as exc: # e.g. an unmapped measure leaves the Observation without its required # code element — reject this row instead of crashing the batch run. raise CmsReportingError( f"Invalid FHIR Observation for {observation.measure_definition.measure_name}: {exc}" ) from exc # The audit row id is stored in the pending marker so polling resume can # recover exactly this observation's payload (a beneficiary may have many # concurrent $report-data submissions). Created BEFORE posting so a failed # request leaves an audit pair too; each retry attempt gets its own row. request_row = CMSApiRequest.objects.create( api_category="data_reporting", endpoint="$report-data", method="POST", request_payload=payload, request_headers={"Content-Type": "application/fhir+json"}, beneficiary=observation.beneficiary, ) try: submit_response = client.post_operation( resource_type="Patient", operation="$report-data", parameters=payload, ) except CmsApiError as exc: # Record the failure on this attempt's audit row, then re-raise so the # batch runner applies its terminal/transient handling. try: note_failed_attempt(request_row, exc) except Exception: # audit recording must not mask the original error logger.warning( "Failed to record failed CMS attempt in the audit trail", exc_info=True, ) raise request_row.status_code = submit_response.pop("_http_status", 200) request_row.save(update_fields=["status_code"]) submission_id = _extract_submission_id(submit_response) if not submission_id: raise CmsReportingError( "Data reporting response missing submission-id for " f"{observation.measure_definition.measure_name} " + describe_response_params(submit_response) ) # Durable tracking; cms_polling_worker polls $submission-status (each # attempt is logged to the audit trail) and applies the final outcome via # apply_reporting_result(). The pending marker keeps this row out of # subsequent batches until the worker overwrites it with the final response. record_submission( api_category=CMSApiRequest.ApiCategory.DATA_REPORTING, endpoint="$report-data", submission_id=submission_id, target_table="OAPMeasureObservation", target_record_id=observation.id, ) observation.cms_submission_response = { "status": "pending_cms_processing", "submission_id": submission_id, "request_id": str(request_row.id), } observation.save(update_fields=["cms_submission_response"]) return { "observation_id": str(observation.id), "measure": observation.measure_definition.measure_name, "status": "submitted", "submission_id": submission_id, } def apply_reporting_result( observation: OAPMeasureObservation, status_response: dict, payload: dict | None = None, ) -> str: """Apply a final $report-data outcome to its OAPMeasureObservation row. Called by cms_polling_worker via the poll_cms_submissions resume command (idempotent: re-applying the same outcome is a no-op). Returns "submitted" or "rejected". """ outcome = _parse_reporting_outcome(status_response) update_fields = ["cms_submission_response", "status"] observation.cms_submission_response = status_response if outcome == "accepted": observation.status = "submitted_to_cms" observation.submission_date = datetime.now(timezone.utc).date() update_fields.append("submission_date") if payload is not None: observation.cms_submission_payload = payload update_fields.append("cms_submission_payload") else: observation.status = "rejected" observation.save(update_fields=update_fields) return "submitted" if outcome == "accepted" else "rejected" _extract_submission_id = CmsClient._extract_submission_id def _parse_reporting_outcome(response: dict) -> str: """Parse data reporting outcome. Returns 'accepted' or 'rejected'. Bodies that validate as FHIR Parameters are walked via the fhirclient model; anything else falls through to raw-dict handling (mock server quirks). """ parsed = parse_fhir_resource(response) if isinstance(parsed, Parameters): for param in parsed.parameter or []: name = (param.name or "").lower() if name == "outcome": for part in param.part or []: code = (part.valueCode or "").lower() value = (part.valueString or "").lower() text = f"{code} {value}" if any(kw in text for kw in ("accept", "success", "complete")): return "accepted" # Nested resource format (mock server): result.resource.parameter[].name=result.valueCodeableConcept.coding[0].code elif name == "result": nested = param.resource if isinstance(nested, Parameters): for np in nested.parameter or []: if (np.name or "") == "result": cc = np.valueCodeableConcept code = ( (cc.coding[0].code or "").lower() if cc and cc.coding else "" ) if any(kw in code for kw in ("accepted", "accept")): return "accepted" else: # Raw fallback — body did not validate as a FHIR resource. for param in response.get("parameter", []): name = param.get("name", "").lower() if name == "outcome": parts = param.get("part", []) for part in parts: code = (part.get("valueCode") or "").lower() value = (part.get("valueString") or "").lower() text = f"{code} {value}" if any(kw in text for kw in ("accept", "success", "complete")): return "accepted" # Nested resource format (mock server): result.resource.parameter[].name=result.valueCodeableConcept.coding[0].code if name == "result": nested = param.get("resource", {}) for np in nested.get("parameter", []): if np.get("name") == "result": coding = np.get("valueCodeableConcept", {}).get("coding", [{}])[ 0 ] code = (coding.get("code") or "").lower() if any(kw in code for kw in ("accepted", "accept")): return "accepted" outcome = CmsClient._extract_operation_outcome(response) if outcome: severity = outcome.get("severity", "") if severity == "informational": return "accepted" return "rejected" def _resolve_measure_key(measure_name: str) -> str: """Map measure name to FHIR mapping key.""" name_lower = measure_name.lower().replace(" ", "_").replace("-", "_") for key in OAP_MEASURE_FHIR_MAPPING: if key.replace("_", "") == name_lower.replace("_", ""): return key if key in name_lower or name_lower in key: return key common_aliases = { "systolic": "blood_pressure_systolic", "diastolic": "blood_pressure_diastolic", "ldl": "ldl_cholesterol", "total_chol": "total_cholesterol", "a1c": "hba1c", "glycosylated_hemoglobin": "hba1c", "gfr": "egfr", "urine_albumin_creatinine_ratio": "uacr", } for alias, key in common_aliases.items(): if alias in name_lower: return key return name_lower