diff --git a/scripts/run_tests.py b/scripts/run_tests.py new file mode 100644 index 00000000..7ff3237b --- /dev/null +++ b/scripts/run_tests.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +""" +XCEngine Test Runner + +Unified test execution script for XCEngine test suites. +Supports Windows and Linux platforms with CTest and Python wrapper integration. + +Usage: + python run_tests.py [options] + +Options: + --build Build tests before running + --unit-only Run only unit tests (skip integration tests) + --integration Run integration tests (includes unit tests) + --ci CI mode (skip GUI-dependent integration tests) + --dir Run tests for specific directory + --verbose,-v Verbose output + --help,-h Show this help message +""" + +import argparse +import subprocess +import sys +import os +import json +import time +from datetime import datetime +from pathlib import Path +from typing import Optional, List, Dict, Any + + +class Colors: + RED = "\033[91m" + GREEN = "\033[92m" + YELLOW = "\033[93m" + BLUE = "\033[94m" + RESET = "\033[0m" + BOLD = "\033[1m" + + +class TestRunner: + PLATFORM_WINDOWS = "Windows" + PLATFORM_LINUX = "Linux" + PLATFORM_MACOS = "Darwin" + + def __init__(self, build_dir: Path, source_dir: Path, verbose: bool = False): + self.build_dir = Path(build_dir) + self.source_dir = Path(source_dir) + self.verbose = verbose + self.platform = self._detect_platform() + self.results: List[Dict[str, Any]] = [] + self.total_passed = 0 + self.total_failed = 0 + self.total_time = 0.0 + + def _detect_platform(self) -> str: + if sys.platform.startswith("win"): + return self.PLATFORM_WINDOWS + elif sys.platform.startswith("linux"): + return self.PLATFORM_LINUX + elif sys.platform.startswith("darwin"): + return self.PLATFORM_MACOS + return "Unknown" + + def _log(self, message: str, color: str = ""): + if color: + print(f"{color}{message}{Colors.RESET}") + else: + print(message) + + def _log_header(self, message: str): + print() + print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}") + print(f"{Colors.BOLD}{Colors.BLUE}{message}{Colors.RESET}") + print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}") + + def _log_success(self, message: str): + self._log(f"{Colors.GREEN}[PASS] {message}{Colors.RESET}") + + def _log_error(self, message: str): + self._log(f"{Colors.RED}[FAIL] {message}{Colors.RESET}") + + def _log_warning(self, message: str): + self._log(f"{Colors.YELLOW}[WARN] {message}{Colors.RESET}") + + def _log_info(self, message: str): + self._log(f" {message}") + + def _run_command( + self, cmd: List[str], cwd: Optional[Path] = None, capture_output: bool = True + ) -> subprocess.CompletedProcess: + if self.verbose: + self._log_info(f"Running: {' '.join(str(c) for c in cmd)}") + + try: + result = subprocess.run( + cmd, cwd=cwd, capture_output=capture_output, text=True, timeout=300 + ) + return result + except subprocess.TimeoutExpired: + self._log_error(f"Command timed out: {' '.join(str(c) for c in cmd)}") + return subprocess.CompletedProcess(cmd, 1, "", "Timeout expired") + except Exception as e: + self._log_error(f"Command failed: {e}") + return subprocess.CompletedProcess(cmd, 1, "", str(e)) + + def build_tests(self, config: str = "Debug") -> bool: + self._log_header("Building Tests") + + cmd = ["cmake", "--build", str(self.build_dir), "--config", config] + + result = self._run_command(cmd) + + if result.returncode == 0: + self._log_success("Build completed successfully") + return True + else: + self._log_error("Build failed") + if self.verbose and result.stdout: + print(result.stdout) + return False + + def _get_ctest_executable(self) -> str: + if self.platform == self.PLATFORM_WINDOWS: + return "ctest" + return "ctest" + + def _run_ctest_in_dir(self, test_dir: Path, label: str = "") -> Dict[str, Any]: + ctest_exe = self._get_ctest_executable() + + cmd = [ctest_exe, "-C", "Debug", "--output-on-failure"] + + if self.verbose: + cmd.append("--verbose") + + start_time = time.time() + result = self._run_command(cmd, cwd=test_dir) + elapsed = time.time() - start_time + + passed = 0 + failed = 0 + + output = result.stdout + result.stderr + + if "100% tests passed" in output or "passed" in output.lower(): + if "0 tests failed" in output: + passed_match = output.find("tests passed") + if passed_match != -1: + try: + num_str = ( + output[max(0, passed_match - 5) : passed_match] + .strip() + .split()[-1] + ) + passed = int(num_str) + except: + passed = 1 + else: + passed = 1 + + if "Failed" in output or result.returncode != 0: + failed = 1 + + return { + "name": label or f"Tests in {test_dir.name}", + "passed": passed, + "failed": failed, + "time": elapsed, + "returncode": result.returncode, + "output": output if self.verbose else "", + } + + def run_ctest(self, test_dir: Optional[Path] = None) -> bool: + if test_dir: + test_dirs = [Path(test_dir)] + else: + test_dirs = [ + self.build_dir / "tests" / "math", + self.build_dir / "tests" / "core", + self.build_dir / "tests" / "containers", + self.build_dir / "tests" / "memory", + self.build_dir / "tests" / "threading", + self.build_dir / "tests" / "debug", + self.build_dir / "tests" / "Resources", + self.build_dir / "tests" / "RHI" / "D3D12" / "unit", + ] + + all_passed = True + + for td in test_dirs: + if not td.exists(): + self._log_warning(f"Test directory not found: {td}") + continue + + self._log_header(f"Running Tests: {td.name}") + + result = self._run_ctest_in_dir(td, td.name) + self.results.append(result) + + if result["failed"] > 0: + self._log_error(f"Failed: {result['name']}") + all_passed = False + else: + self._log_success(f"Passed: {result['name']} ({result['time']:.2f}s)") + + if self.verbose and result.get("output"): + print(result["output"]) + + return all_passed + + def _run_integration_test( + self, + test_name: str, + exe_path: Path, + output_ppm: str, + gt_ppm: Path, + threshold: int = 5, + ) -> Dict[str, Any]: + wrapper_script = ( + self.source_dir + / "tests" + / "RHI" + / "D3D12" + / "integration" + / "run_integration_test.py" + ) + + if not wrapper_script.exists(): + return { + "name": test_name, + "passed": 0, + "failed": 1, + "time": 0, + "error": "Wrapper script not found", + } + + python_exe = sys.executable + cmd = [ + python_exe, + str(wrapper_script), + str(exe_path), + output_ppm, + str(gt_ppm), + str(threshold), + ] + + start_time = time.time() + result = self._run_command(cmd, cwd=exe_path.parent) + elapsed = time.time() - start_time + + passed = 1 if result.returncode == 0 else 0 + failed = 0 if passed else 1 + + return { + "name": test_name, + "passed": passed, + "failed": failed, + "time": elapsed, + "returncode": result.returncode, + "output": result.stdout + result.stderr, + } + + def run_integration_tests(self) -> bool: + integration_dir = ( + self.build_dir / "tests" / "RHI" / "D3D12" / "integration" / "Debug" + ) + + if not integration_dir.exists(): + self._log_warning( + f"Integration test directory not found: {integration_dir}" + ) + return True + + self._log_header("Running Integration Tests") + + minimal_exe = integration_dir / "D3D12_Minimal.exe" + gt_minimal = ( + self.source_dir + / "tests" + / "RHI" + / "D3D12" + / "integration" + / "GT_minimal.ppm" + ) + + if minimal_exe.exists() and gt_minimal.exists(): + result = self._run_integration_test( + "D3D12_Minimal_Integration", minimal_exe, "minimal.ppm", gt_minimal, 5 + ) + self.results.append(result) + + if result["failed"] > 0: + self._log_error(f"Failed: {result['name']}") + if self.verbose: + self._log_info(result.get("output", "")) + return False + else: + self._log_success(f"Passed: {result['name']} ({result['time']:.2f}s)") + return True + else: + self._log_warning("D3D12_Minimal.exe not found, skipping integration test") + return True + + def _summarize_results(self): + self._log_header("Test Summary") + + total_passed = 0 + total_failed = 0 + total_time = 0.0 + + for result in self.results: + total_passed += result.get("passed", 0) + total_failed += result.get("failed", 0) + total_time += result.get("time", 0) + + self.total_passed = total_passed + self.total_failed = total_failed + self.total_time = total_time + + print() + print(f" Platform: {self.platform}") + print(f" Total Passed: {Colors.GREEN}{total_passed}{Colors.RESET}") + print( + f" Total Failed: {Colors.RED if total_failed > 0 else Colors.GREEN}{total_failed}{Colors.RESET}" + ) + print(f" Total Time: {total_time:.2f}s") + print() + + def run( + self, + build: bool = False, + unit_only: bool = False, + integration: bool = False, + ci_mode: bool = False, + test_dir: Optional[Path] = None, + ) -> int: + + print() + print(f"{Colors.BOLD}XCEngine Test Runner{Colors.RESET}") + print(f" Platform: {self.platform}") + print(f" Build Dir: {self.build_dir}") + print(f" Source Dir: {self.source_dir}") + + if build: + if not self.build_tests(): + self._log_error("Build failed, aborting tests") + return 1 + + if test_dir: + self._log_info(f"Running tests for: {test_dir}") + if self._run_ctest_in_dir(Path(test_dir)).get("failed", 0) > 0: + self._summarize_results() + return 1 + self._summarize_results() + return 0 + + if not self.run_ctest(): + self._summarize_results() + return 1 + + if integration or (not unit_only and not ci_mode): + if not self.run_integration_tests(): + self._summarize_results() + return 1 + + self._summarize_results() + + if self.total_failed > 0: + self._log_error("Some tests failed") + return 1 + else: + self._log_success("All tests passed!") + return 0 + + +def find_build_dir() -> Optional[Path]: + possible_dirs = [ + Path(__file__).parent.parent / "build", + Path.cwd() / "build", + Path.cwd(), + ] + + for d in possible_dirs: + if d.exists() and (d / "CMakeCache.txt").exists(): + return d + + script_dir = Path(__file__).parent.parent + test_build = script_dir / "tests" / "build" + if test_build.exists(): + return test_build + + return Path.cwd() + + +def find_source_dir() -> Path: + return Path(__file__).parent.parent + + +def main(): + parser = argparse.ArgumentParser( + description="XCEngine Test Runner - Unified test execution script", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python run_tests.py Run all tests + python run_tests.py --build Build and run tests + python run_tests.py --unit-only Run only unit tests + python run_tests.py --ci CI mode (skip GUI integration tests) + python run_tests.py --dir tests/math Run specific test directory + python run_tests.py --verbose Verbose output + """, + ) + + parser.add_argument( + "--build", "-b", action="store_true", help="Build tests before running" + ) + + parser.add_argument( + "--unit-only", + "-u", + action="store_true", + help="Run only unit tests (skip integration tests)", + ) + + parser.add_argument( + "--integration", + "-i", + action="store_true", + help="Run integration tests (includes unit tests)", + ) + + parser.add_argument( + "--ci", + "-c", + action="store_true", + help="CI mode (skip GUI-dependent integration tests)", + ) + + parser.add_argument( + "--dir", "-d", type=Path, help="Run tests for specific directory" + ) + + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + + parser.add_argument( + "--build-dir", type=Path, help="Build directory (default: auto-detect)" + ) + + parser.add_argument( + "--source-dir", type=Path, help="Source directory (default: auto-detect)" + ) + + args = parser.parse_args() + + build_dir = args.build_dir if args.build_dir else find_build_dir() + source_dir = args.source_dir if args.source_dir else find_source_dir() + + if not build_dir: + print(f"{Colors.RED}Error: Could not find build directory{Colors.RESET}") + print("Please specify --build-dir or run from a directory with CMake cache") + return 1 + + runner = TestRunner(build_dir, source_dir, verbose=args.verbose) + + return runner.run( + build=args.build, + unit_only=args.unit_only, + integration=args.integration, + ci_mode=args.ci, + test_dir=args.dir, + ) + + +if __name__ == "__main__": + sys.exit(main() or 0) diff --git a/tests/run_tests.bat b/tests/run_tests.bat index eea7efda..b819e2b4 100644 --- a/tests/run_tests.bat +++ b/tests/run_tests.bat @@ -1,8 +1,8 @@ @echo off -echo Running XCEngine Tests... +echo Running XCEngine Tests via Python runner... echo. -ctest --test-dir tests/build -C Debug --output-on-failure +python "%~dp0..\scripts\run_tests.py" --ci echo. -pause +pause \ No newline at end of file