Databricks Disaster Recovery Kit

scripts/delta_replication.py"""Delta Lake Cross-Region Replication for Disaster Recovery. Databricks notebook for replicating Delta tables between primary and DR regions using deep clone operations and streaming Change Data Feed (CDF) sync. Supports: - Full deep clone for initial replication and periodic refresh - Streaming CDF-based incremental replication for near-real-time sync - Parallel table replication with configurable concurrency - Replication lag monitoring and alerting - Data validation between source and target Usage: Configure the widgets at the top of the notebook, then run all cells. Can also be scheduled as a Databricks workflow job. Datanest Digital — https://datanest.dev """ from __future__ import annotations import json import logging import time from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from typing import Any, Optional # Databricks notebook environment provides `spark` and `dbutils` globally. # These imports are for type hints only — they resolve at runtime on the cluster. try: from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as F except ImportError: pass # Running outside Databricks; types only logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("delta_replication")