Spark Performance Masterclass

benchmarks/benchmark_runner.py"""Spark Performance Benchmark Runner. A framework for measuring Spark job performance metrics including execution time, stage count, shuffle bytes, spill bytes, and task count. Designed to run on Databricks clusters with DBR 14+. Copyright (c) 2026 DataStack Pro. MIT License. """ from __future__ import annotations import json import time from dataclasses import dataclass, field, asdict from datetime import datetime from typing import Any, Callable from pyspark.sql import SparkSession, DataFrame @dataclass class BenchmarkResult: """Container for benchmark execution results. Attributes: name: Benchmark name identifier. category: Benchmark category (join, io, aggregation, delta). description: Human-readable description of what was benchmarked. execution_time_seconds: Wall-clock execution time. num_stages: Number of Spark stages in the job. num_tasks: Total number of tasks across all stages. shuffle_read_bytes: Total shuffle read bytes across all stages. shuffle_write_bytes: Total shuffle write bytes across all stages. spill_memory_bytes: Total memory spill bytes. spill_disk_bytes: Total disk spill bytes. input_rows: Number of input rows processed. output_rows: Number of output rows produced. config: Spark configuration used for this benchmark. metadata: Additional metadata about the benchmark run. timestamp: ISO format timestamp of when the benchmark ran.