"""Generate benchmark report from evaluation metrics. This script runs policy evaluation on nightly runs and generates a static HTML dashboard for tracking policy performance over time. """ from __future__ import annotations import json import shutil from datetime import datetime from pathlib import Path import tyro import wandb import mjlab from mjlab.tasks.tracking.scripts.evaluate import EvaluateConfig, run_evaluate # Metrics to display: (key, label, unit, scale, higher_is_better) METRICS = [ ("success_rate", "Success Rate", "%", 100, True), ("mpkpe", "MPKPE", "m", 1, False), ("r_mpkpe", "R-MPKPE", "m", 1, False), ("ee_pos_error", "EE Position Error", "m", 1, False), ("ee_ori_error", "EE Orientation Error", "rad", 1, False), ("joint_vel_error", "Joint Velocity Error", "rad/s", 1, False), ] def evaluate_run(run_path: str, num_envs: int = 1024) -> dict: """Evaluate a single run and return metrics with metadata.""" api = wandb.Api() run = api.run(run_path) print(f"Evaluating run: {run.name} ({run.id})") cfg = EvaluateConfig(wandb_run_path=run_path, num_envs=num_envs) metrics = run_evaluate("Mjlab-Tracking-Flat-Unitree-G1", cfg) # Get commit SHA from run metadata. commit = run.commit or run.config.get("commit", "unknown") return { "id": run.id, "name": run.name, "url": run.url, "created_at": run.created_at, "commit": commit[:7] if len(commit) > 7 else commit, "metrics": metrics, } def load_throughput_data(output_dir: Path) -> list[dict]: """Load throughput benchmark data if available.""" data_file = output_dir / "throughput_data.json" if not data_file.exists(): return [] with open(data_file) as f: return json.load(f) def generate_html_report(runs: list[dict], output_dir: Path) -> None: """Generate static HTML dashboard from evaluation data.""" output_dir.mkdir(parents=True, exist_ok=True) # Save raw data. with open(output_dir / "data.json", "w") as f: json.dump(runs, f, indent=2, default=str) # Copy task images for the throughput dashboard. images_src = Path(__file__).parent / "nightly_images" if images_src.is_dir(): images_dst = output_dir / "images" if images_dst.exists(): shutil.rmtree(images_dst) shutil.copytree(images_src, images_dst) # Load throughput data if available. throughput_data = load_throughput_data(output_dir) html = generate_dashboard_html(runs, throughput_data) with open(output_dir / "index.html", "w") as f: f.write(html) print(f"Report generated at {output_dir / 'index.html'}") def generate_dashboard_html(runs: list[dict], throughput_data: list[dict]) -> str: """Generate the HTML dashboard content.""" runs_json = json.dumps(runs, default=str) metrics_json = json.dumps(METRICS) throughput_json = json.dumps(throughput_data, default=str) timestamp = datetime.now().strftime("%Y-%m-%d %H:%M") github_repo = "https://github.com/mujocolab/mjlab" return f""" mjlab Nightly Benchmark

mjlab Nightly Benchmark

Performance tracking over time

Updated: {timestamp}

Nightly motion imitation training and evaluation on Unitree G1 (1024 trials per run).

Physics simulation throughput across tasks (4096 parallel envs, NVIDIA RTX 5090).

""" def load_cached_results(output_dir: Path) -> dict[str, dict]: """Load previously evaluated results from cache.""" data_file = output_dir / "data.json" if not data_file.exists(): return {} with open(data_file) as f: runs = json.load(f) return {run["id"]: run for run in runs} def main( run_paths: list[str] | None = None, entity: str = "gcbc_researchers", project: str = "mjlab", tag: str = "nightly", eval_limit: int = 0, num_envs: int = 1024, output_dir: Path = Path("benchmark_results"), ) -> None: """Generate benchmark report by evaluating nightly runs. Args: run_paths: Specific run paths to evaluate (entity/project/run_id). entity: WandB entity. project: WandB project name. tag: Filter runs by tag. eval_limit: Maximum number of NEW runs to evaluate per invocation (0 = no limit). num_envs: Number of envs for evaluation. output_dir: Output directory for generated report. """ # Load cached results to avoid re-evaluating old runs. cached = load_cached_results(output_dir) print(f"Loaded {len(cached)} cached evaluation results") # Start with all cached results (preserves historical data). eval_results_by_id: dict[str, dict] = dict(cached) new_evals = 0 if run_paths: for run_path in run_paths: run_id = run_path.split("/")[-1] if run_id in eval_results_by_id: print(f"Using cached result for {run_id}") else: try: result = evaluate_run(run_path, num_envs) except RuntimeError as e: print(f"Skipping {run_path}: {e}") continue eval_results_by_id[run_id] = result new_evals += 1 else: api = wandb.Api() print(f"Fetching runs from {entity}/{project} with tag '{tag}'...") runs = api.runs(f"{entity}/{project}", filters={"tags": tag}, order="-created_at") for run in runs: if run.state != "finished": continue if run.id in eval_results_by_id: print(f"Using cached result for {run.name} ({run.id})") else: if eval_limit > 0 and new_evals >= eval_limit: print(f"Reached eval limit ({eval_limit}), skipping remaining new runs") break run_path = f"{entity}/{project}/{run.id}" try: result = evaluate_run(run_path, num_envs) except RuntimeError as e: print(f"Skipping {run.name} ({run.id}): {e}") continue eval_results_by_id[run.id] = result new_evals += 1 eval_results = list(eval_results_by_id.values()) print(f"Total runs: {len(eval_results)} ({new_evals} newly evaluated)") generate_html_report(eval_results, output_dir) if __name__ == "__main__": tyro.cli(main, config=mjlab.TYRO_FLAGS)