486 lines
15 KiB
Python
486 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
XCEngine Test Runner
|
|
|
|
Unified test execution script for XCEngine test suites.
|
|
Supports Windows and Linux platforms with CTest and Python wrapper integration.
|
|
|
|
Usage:
|
|
python run_tests.py [options]
|
|
|
|
Options:
|
|
--build Build tests before running
|
|
--unit-only Run only unit tests (skip integration tests)
|
|
--integration Run integration tests (includes unit tests)
|
|
--ci CI mode (skip GUI-dependent integration tests)
|
|
--dir <path> Run tests for specific directory
|
|
--verbose,-v Verbose output
|
|
--help,-h Show this help message
|
|
"""
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
import os
|
|
import json
|
|
import time
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional, List, Dict, Any
|
|
|
|
|
|
class Colors:
|
|
RED = "\033[91m"
|
|
GREEN = "\033[92m"
|
|
YELLOW = "\033[93m"
|
|
BLUE = "\033[94m"
|
|
RESET = "\033[0m"
|
|
BOLD = "\033[1m"
|
|
|
|
|
|
class TestRunner:
|
|
PLATFORM_WINDOWS = "Windows"
|
|
PLATFORM_LINUX = "Linux"
|
|
PLATFORM_MACOS = "Darwin"
|
|
|
|
def __init__(self, build_dir: Path, source_dir: Path, verbose: bool = False):
|
|
self.build_dir = Path(build_dir)
|
|
self.source_dir = Path(source_dir)
|
|
self.verbose = verbose
|
|
self.platform = self._detect_platform()
|
|
self.results: List[Dict[str, Any]] = []
|
|
self.total_passed = 0
|
|
self.total_failed = 0
|
|
self.total_time = 0.0
|
|
|
|
def _detect_platform(self) -> str:
|
|
if sys.platform.startswith("win"):
|
|
return self.PLATFORM_WINDOWS
|
|
elif sys.platform.startswith("linux"):
|
|
return self.PLATFORM_LINUX
|
|
elif sys.platform.startswith("darwin"):
|
|
return self.PLATFORM_MACOS
|
|
return "Unknown"
|
|
|
|
def _log(self, message: str, color: str = ""):
|
|
if color:
|
|
print(f"{color}{message}{Colors.RESET}")
|
|
else:
|
|
print(message)
|
|
|
|
def _log_header(self, message: str):
|
|
print()
|
|
print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}")
|
|
print(f"{Colors.BOLD}{Colors.BLUE}{message}{Colors.RESET}")
|
|
print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}")
|
|
|
|
def _log_success(self, message: str):
|
|
self._log(f"{Colors.GREEN}[PASS] {message}{Colors.RESET}")
|
|
|
|
def _log_error(self, message: str):
|
|
self._log(f"{Colors.RED}[FAIL] {message}{Colors.RESET}")
|
|
|
|
def _log_warning(self, message: str):
|
|
self._log(f"{Colors.YELLOW}[WARN] {message}{Colors.RESET}")
|
|
|
|
def _log_info(self, message: str):
|
|
self._log(f" {message}")
|
|
|
|
def _run_command(
|
|
self, cmd: List[str], cwd: Optional[Path] = None, capture_output: bool = True
|
|
) -> subprocess.CompletedProcess:
|
|
if self.verbose:
|
|
self._log_info(f"Running: {' '.join(str(c) for c in cmd)}")
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
cmd, cwd=cwd, capture_output=capture_output, text=True, timeout=300
|
|
)
|
|
return result
|
|
except subprocess.TimeoutExpired:
|
|
self._log_error(f"Command timed out: {' '.join(str(c) for c in cmd)}")
|
|
return subprocess.CompletedProcess(cmd, 1, "", "Timeout expired")
|
|
except Exception as e:
|
|
self._log_error(f"Command failed: {e}")
|
|
return subprocess.CompletedProcess(cmd, 1, "", str(e))
|
|
|
|
def build_tests(self, config: str = "Debug") -> bool:
|
|
self._log_header("Building Tests")
|
|
|
|
cmd = ["cmake", "--build", str(self.build_dir), "--config", config]
|
|
|
|
result = self._run_command(cmd)
|
|
|
|
if result.returncode == 0:
|
|
self._log_success("Build completed successfully")
|
|
return True
|
|
else:
|
|
self._log_error("Build failed")
|
|
if self.verbose and result.stdout:
|
|
print(result.stdout)
|
|
return False
|
|
|
|
def _get_ctest_executable(self) -> str:
|
|
if self.platform == self.PLATFORM_WINDOWS:
|
|
return "ctest"
|
|
return "ctest"
|
|
|
|
def _run_ctest_in_dir(self, test_dir: Path, label: str = "") -> Dict[str, Any]:
|
|
ctest_exe = self._get_ctest_executable()
|
|
|
|
cmd = [ctest_exe, "-C", "Debug", "--output-on-failure"]
|
|
|
|
if self.verbose:
|
|
cmd.append("--verbose")
|
|
|
|
start_time = time.time()
|
|
result = self._run_command(cmd, cwd=test_dir)
|
|
elapsed = time.time() - start_time
|
|
|
|
passed = 0
|
|
failed = 0
|
|
|
|
output = result.stdout + result.stderr
|
|
|
|
if "100% tests passed" in output or "passed" in output.lower():
|
|
if "0 tests failed" in output:
|
|
passed_match = output.find("tests passed")
|
|
if passed_match != -1:
|
|
try:
|
|
num_str = (
|
|
output[max(0, passed_match - 5) : passed_match]
|
|
.strip()
|
|
.split()[-1]
|
|
)
|
|
passed = int(num_str)
|
|
except:
|
|
passed = 1
|
|
else:
|
|
passed = 1
|
|
|
|
if "Failed" in output or result.returncode != 0:
|
|
failed = 1
|
|
|
|
return {
|
|
"name": label or f"Tests in {test_dir.name}",
|
|
"passed": passed,
|
|
"failed": failed,
|
|
"time": elapsed,
|
|
"returncode": result.returncode,
|
|
"output": output if self.verbose else "",
|
|
}
|
|
|
|
def run_ctest(self, test_dir: Optional[Path] = None) -> bool:
|
|
if test_dir:
|
|
test_dirs = [Path(test_dir)]
|
|
else:
|
|
test_dirs = [
|
|
self.build_dir / "tests" / "math",
|
|
self.build_dir / "tests" / "core",
|
|
self.build_dir / "tests" / "containers",
|
|
self.build_dir / "tests" / "memory",
|
|
self.build_dir / "tests" / "threading",
|
|
self.build_dir / "tests" / "debug",
|
|
self.build_dir / "tests" / "Resources",
|
|
self.build_dir / "tests" / "RHI" / "D3D12" / "unit",
|
|
]
|
|
|
|
all_passed = True
|
|
|
|
for td in test_dirs:
|
|
if not td.exists():
|
|
self._log_warning(f"Test directory not found: {td}")
|
|
continue
|
|
|
|
self._log_header(f"Running Tests: {td.name}")
|
|
|
|
result = self._run_ctest_in_dir(td, td.name)
|
|
self.results.append(result)
|
|
|
|
if result["failed"] > 0:
|
|
self._log_error(f"Failed: {result['name']}")
|
|
all_passed = False
|
|
else:
|
|
self._log_success(f"Passed: {result['name']} ({result['time']:.2f}s)")
|
|
|
|
if self.verbose and result.get("output"):
|
|
print(result["output"])
|
|
|
|
return all_passed
|
|
|
|
def _run_integration_test(
|
|
self,
|
|
test_name: str,
|
|
exe_path: Path,
|
|
output_ppm: str,
|
|
gt_ppm: Path,
|
|
threshold: int = 5,
|
|
) -> Dict[str, Any]:
|
|
wrapper_script = (
|
|
self.source_dir
|
|
/ "tests"
|
|
/ "RHI"
|
|
/ "D3D12"
|
|
/ "integration"
|
|
/ "run_integration_test.py"
|
|
)
|
|
|
|
if not wrapper_script.exists():
|
|
return {
|
|
"name": test_name,
|
|
"passed": 0,
|
|
"failed": 1,
|
|
"time": 0,
|
|
"error": "Wrapper script not found",
|
|
}
|
|
|
|
python_exe = sys.executable
|
|
cmd = [
|
|
python_exe,
|
|
str(wrapper_script),
|
|
str(exe_path),
|
|
output_ppm,
|
|
str(gt_ppm),
|
|
str(threshold),
|
|
]
|
|
|
|
start_time = time.time()
|
|
result = self._run_command(cmd, cwd=exe_path.parent)
|
|
elapsed = time.time() - start_time
|
|
|
|
passed = 1 if result.returncode == 0 else 0
|
|
failed = 0 if passed else 1
|
|
|
|
return {
|
|
"name": test_name,
|
|
"passed": passed,
|
|
"failed": failed,
|
|
"time": elapsed,
|
|
"returncode": result.returncode,
|
|
"output": result.stdout + result.stderr,
|
|
}
|
|
|
|
def run_integration_tests(self) -> bool:
|
|
integration_dir = self.build_dir / "tests" / "RHI" / "D3D12" / "integration"
|
|
integration_src_dir = (
|
|
self.source_dir / "tests" / "RHI" / "D3D12" / "integration"
|
|
)
|
|
|
|
if not integration_dir.exists():
|
|
self._log_warning(
|
|
f"Integration test directory not found: {integration_dir}"
|
|
)
|
|
return True
|
|
|
|
self._log_header("Running Integration Tests")
|
|
|
|
integration_tests = [
|
|
("D3D12_Minimal_Integration", "minimal", "minimal.ppm"),
|
|
("D3D12_Sphere_Integration", "sphere", "sphere.ppm"),
|
|
("D3D12_Triangle_Integration", "triangle", "triangle.ppm"),
|
|
("D3D12_Quad_Integration", "quad", "quad.ppm"),
|
|
]
|
|
|
|
all_passed = True
|
|
for test_name, test_folder, output_ppm in integration_tests:
|
|
exe_path = (
|
|
integration_dir
|
|
/ test_folder
|
|
/ "Debug"
|
|
/ f"D3D12_{test_folder.title()}.exe"
|
|
)
|
|
gt_ppm = integration_src_dir / test_folder / "GT.ppm"
|
|
|
|
if exe_path.exists() and gt_ppm.exists():
|
|
result = self._run_integration_test(
|
|
test_name, exe_path, output_ppm, gt_ppm, 0
|
|
)
|
|
self.results.append(result)
|
|
|
|
if result["failed"] > 0:
|
|
self._log_error(f"Failed: {result['name']}")
|
|
if self.verbose:
|
|
self._log_info(result.get("output", ""))
|
|
all_passed = False
|
|
else:
|
|
self._log_success(
|
|
f"Passed: {result['name']} ({result['time']:.2f}s)"
|
|
)
|
|
else:
|
|
self._log_warning(f"{test_name} not found, skipping")
|
|
|
|
return all_passed
|
|
|
|
def _summarize_results(self):
|
|
self._log_header("Test Summary")
|
|
|
|
total_passed = 0
|
|
total_failed = 0
|
|
total_time = 0.0
|
|
|
|
for result in self.results:
|
|
total_passed += result.get("passed", 0)
|
|
total_failed += result.get("failed", 0)
|
|
total_time += result.get("time", 0)
|
|
|
|
self.total_passed = total_passed
|
|
self.total_failed = total_failed
|
|
self.total_time = total_time
|
|
|
|
print()
|
|
print(f" Platform: {self.platform}")
|
|
print(f" Total Passed: {Colors.GREEN}{total_passed}{Colors.RESET}")
|
|
print(
|
|
f" Total Failed: {Colors.RED if total_failed > 0 else Colors.GREEN}{total_failed}{Colors.RESET}"
|
|
)
|
|
print(f" Total Time: {total_time:.2f}s")
|
|
print()
|
|
|
|
def run(
|
|
self,
|
|
build: bool = False,
|
|
unit_only: bool = False,
|
|
integration: bool = False,
|
|
ci_mode: bool = False,
|
|
test_dir: Optional[Path] = None,
|
|
) -> int:
|
|
|
|
print()
|
|
print(f"{Colors.BOLD}XCEngine Test Runner{Colors.RESET}")
|
|
print(f" Platform: {self.platform}")
|
|
print(f" Build Dir: {self.build_dir}")
|
|
print(f" Source Dir: {self.source_dir}")
|
|
|
|
if build:
|
|
if not self.build_tests():
|
|
self._log_error("Build failed, aborting tests")
|
|
return 1
|
|
|
|
if test_dir:
|
|
self._log_info(f"Running tests for: {test_dir}")
|
|
if self._run_ctest_in_dir(Path(test_dir)).get("failed", 0) > 0:
|
|
self._summarize_results()
|
|
return 1
|
|
self._summarize_results()
|
|
return 0
|
|
|
|
if not self.run_ctest():
|
|
self._summarize_results()
|
|
return 1
|
|
|
|
if integration or (not unit_only and not ci_mode):
|
|
if not self.run_integration_tests():
|
|
self._summarize_results()
|
|
return 1
|
|
|
|
self._summarize_results()
|
|
|
|
if self.total_failed > 0:
|
|
self._log_error("Some tests failed")
|
|
return 1
|
|
else:
|
|
self._log_success("All tests passed!")
|
|
return 0
|
|
|
|
|
|
def find_build_dir() -> Optional[Path]:
|
|
possible_dirs = [
|
|
Path(__file__).parent.parent / "build",
|
|
Path.cwd() / "build",
|
|
Path.cwd(),
|
|
]
|
|
|
|
for d in possible_dirs:
|
|
if d.exists() and (d / "CMakeCache.txt").exists():
|
|
return d
|
|
|
|
script_dir = Path(__file__).parent.parent
|
|
test_build = script_dir / "tests" / "build"
|
|
if test_build.exists():
|
|
return test_build
|
|
|
|
return Path.cwd()
|
|
|
|
|
|
def find_source_dir() -> Path:
|
|
return Path(__file__).parent.parent
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="XCEngine Test Runner - Unified test execution script",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
python run_tests.py Run all tests
|
|
python run_tests.py --build Build and run tests
|
|
python run_tests.py --unit-only Run only unit tests
|
|
python run_tests.py --ci CI mode (skip GUI integration tests)
|
|
python run_tests.py --dir tests/math Run specific test directory
|
|
python run_tests.py --verbose Verbose output
|
|
""",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--build", "-b", action="store_true", help="Build tests before running"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--unit-only",
|
|
"-u",
|
|
action="store_true",
|
|
help="Run only unit tests (skip integration tests)",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--integration",
|
|
"-i",
|
|
action="store_true",
|
|
help="Run integration tests (includes unit tests)",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--ci",
|
|
"-c",
|
|
action="store_true",
|
|
help="CI mode (skip GUI-dependent integration tests)",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--dir", "-d", type=Path, help="Run tests for specific directory"
|
|
)
|
|
|
|
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
|
|
|
parser.add_argument(
|
|
"--build-dir", type=Path, help="Build directory (default: auto-detect)"
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--source-dir", type=Path, help="Source directory (default: auto-detect)"
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
build_dir = args.build_dir if args.build_dir else find_build_dir()
|
|
source_dir = args.source_dir if args.source_dir else find_source_dir()
|
|
|
|
if not build_dir:
|
|
print(f"{Colors.RED}Error: Could not find build directory{Colors.RESET}")
|
|
print("Please specify --build-dir or run from a directory with CMake cache")
|
|
return 1
|
|
|
|
runner = TestRunner(build_dir, source_dir, verbose=args.verbose)
|
|
|
|
return runner.run(
|
|
build=args.build,
|
|
unit_only=args.unit_only,
|
|
integration=args.integration,
|
|
ci_mode=args.ci,
|
|
test_dir=args.dir,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main() or 0)
|