run_eval_cli.py

#!/usr/bin/env python
"""
Local evaluation script for CLI model.
This script runs evaluation locally without cluster job submission.

Usage:
    python run_eval_cli.py --model_path PATH --eval_tasks TASK1,TASK2 [options]
"""

import argparse
import os
import subprocess
import sys


def parse_args():
    parser = argparse.ArgumentParser(description="Local CLI Evaluation Script")
    parser.add_argument("--num_machines", type=int, default=1, help="Number of machines")
    parser.add_argument("--num_gpus", type=int, default=1, help="Number of GPUs per machine")
    parser.add_argument("--pretrain_vlm_model", type=str,
                        default="llava-onevision-qwen2-0.5b-mid-stage-a4",
                        help="Pretrained VLM model name")
    parser.add_argument("--vlm_exp_layers", type=str, default="RANGE-1-24-4",
                        help="VLM experiment layers")
    parser.add_argument("--vision_exp_layers", type=str, default="RANGE-1-28-4",
                        help="Vision experiment layers")
    parser.add_argument("--projector_name", type=str, default="mlp2x_gelu",
                        help="Projector name")
    parser.add_argument("--model_path", type=str, default='/ainative/muti-modal/yuhang/438262/projects/vlm/models/finetune/CC-llava-onevision-qwen2-0.5b-mid-stage-a4-mlp2x_gelu-NAMEVisualSkipMultiProjectorVisionSumAdaptiveWeightTVSimwPMAMultiQueryLoraProjector128Alpha128FixOriProjector_InitProj_20Percent_22-LayerRANGE-1-24-2-VisionRANGE-1-28-2-bs256-32ppus',
                        help="Path to the trained model checkpoint")
    parser.add_argument("--output_path", type=str, default=None,
                        help="Output path for evaluation results (default: same as model_path)")
    parser.add_argument("--eval_tasks", type=str,
                        default="ai2d",
                        help="Comma-separated list of evaluation tasks")
    parser.add_argument("--init_env", action="store_true",
                        help="Run environment initialization script")
    return parser.parse_args()


# Available evaluation tasks
AVAILABLE_TASKS = [
    # Common benchmarks
    "mmbench_en_dev", "ai2d", "chartqa", "docvqa_val", "infovqa_val",
    # Math/reasoning
    "mathvista_testmini", "mathverse_testmini_vision_only",
    "mathverse_testmini_vision_dominant", "mathverse_testmini_vision_intensive",
    # General VQA
    "mme", "infovqa_test", "vizwiz_vqa_val", "pope",
    # Multi-modal understanding
    "mmvet", "mmmu", "mmstar", "seedbench", "scienceqa_img",
    # LLaVA benchmarks
    "llava_in_the_wild", "llava_wilder_small",
    # Real-world QA
    "realworldqa", "docvqa_test", "gqa", "ok_vqa",
]


def run_single_eval(args, eval_task, env, script_dir):
    """Run evaluation for a single task."""
    eval_output_path = os.path.join(args.output_path, f"TestRelease-{eval_task}_results")
    os.makedirs(eval_output_path, exist_ok=True)

    # Update environment for this task
    task_env = env.copy()
    task_env["EVAL_TASK_LIST"] = eval_task
    task_env["EVAL_OUTPUT_PATH"] = eval_output_path

    print(f"\nRunning evaluation for: {eval_task}")
    print(f"Output path: {eval_output_path}")

    eval_script = os.path.join(script_dir, "scripts/eval/eval_single.sh")
    if not os.path.exists(eval_script):
        print(f"Error: Evaluation script not found at {eval_script}")
        return False

    result = subprocess.run(["bash", eval_script], env=task_env)
    return result.returncode == 0


def main():
    args = parse_args()

    # Validate model path
    if not os.path.exists(args.model_path):
        print(f"Error: Model path does not exist: {args.model_path}")
        sys.exit(1)

    # Set output path
    if args.output_path is None:
        args.output_path = args.model_path
    os.makedirs(args.output_path, exist_ok=True)

    # Parse evaluation tasks
    eval_tasks = [t.strip() for t in args.eval_tasks.split(",") if t.strip()]

    # Validate tasks
    invalid_tasks = [t for t in eval_tasks if t not in AVAILABLE_TASKS]
    if invalid_tasks:
        print(f"Warning: Unknown evaluation tasks: {invalid_tasks}")
        print(f"Available tasks: {AVAILABLE_TASKS}")

    # Set environment variables
    env = os.environ.copy()
    env.update({
        "NUM_MACHINES": str(args.num_machines),
        "NUM_GPUS": str(args.num_gpus),
        "VLM_EXP_LAYERS": args.vlm_exp_layers,
        "SAVE_PATH": args.model_path,
        "PRETRAIN_VLM_MODEL": args.pretrain_vlm_model,
        "VISION_EXP_LAYERS": args.vision_exp_layers,
    })

    print("=" * 60)
    print("CLI Evaluation Configuration")
    print("=" * 60)
    print(f"Model Path: {args.model_path}")
    print(f"Output Path: {args.output_path}")
    print(f"Num Machines: {args.num_machines}")
    print(f"Num GPUs: {args.num_gpus}")
    print(f"Pretrain VLM Model: {args.pretrain_vlm_model}")
    print(f"VLM Exp Layers: {args.vlm_exp_layers}")
    print(f"Vision Exp Layers: {args.vision_exp_layers}")
    print(f"Evaluation Tasks: {eval_tasks}")
    print("=" * 60)

    script_dir = os.path.dirname(os.path.abspath(__file__))

    # Optionally run init script
    if args.init_env:
        init_script = os.path.join(script_dir, "scripts/env/init_ppu_eval.sh")
        if os.path.exists(init_script):
            print("Running environment initialization...")
            subprocess.run(["bash", init_script], env=env, check=True)

    # Run evaluations
    results = {}
    for task in eval_tasks:
        success = run_single_eval(args, task, env, script_dir)
        results[task] = "Success" if success else "Failed"

    # Print summary
    print("\n" + "=" * 60)
    print("Evaluation Summary")
    print("=" * 60)
    for task, status in results.items():
        print(f"  {task}: {status}")
    print("=" * 60)

    # Exit with error if any evaluation failed
    if "Failed" in results.values():
        sys.exit(1)


if __name__ == "__main__":
    main()