refactor: move run_tests.py to tests/ and fix integration test GT.ppm paths
This commit is contained in:
@@ -29,12 +29,6 @@ target_link_libraries(D3D12_Minimal PRIVATE
|
||||
XCEngine
|
||||
)
|
||||
|
||||
add_custom_command(TARGET D3D12_Minimal POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy_directory
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/Res
|
||||
$<TARGET_FILE_DIR:D3D12_Minimal>/Res
|
||||
)
|
||||
|
||||
add_custom_command(TARGET D3D12_Minimal POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy_if_different
|
||||
${CMAKE_SOURCE_DIR}/tests/RHI/D3D12/integration/compare_ppm.py
|
||||
|
||||
@@ -38,15 +38,7 @@ target_include_directories(d3d12_engine_tests PRIVATE
|
||||
${PROJECT_ROOT_DIR}/engine/src
|
||||
)
|
||||
|
||||
target_compile_definitions(d3d12_engine_tests PRIVATE
|
||||
TEST_RESOURCES_DIR="${PROJECT_ROOT_DIR}/tests/RHI/D3D12/integration/minimal/Res"
|
||||
)
|
||||
|
||||
add_custom_command(TARGET d3d12_engine_tests POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E copy_directory
|
||||
${PROJECT_ROOT_DIR}/tests/RHI/D3D12/integration/minimal/Res
|
||||
$<TARGET_FILE_DIR:d3d12_engine_tests>/Res
|
||||
)
|
||||
|
||||
enable_testing()
|
||||
add_test(NAME D3D12EngineTests COMMAND d3d12_engine_tests)
|
||||
|
||||
485
tests/run_tests.py
Normal file
485
tests/run_tests.py
Normal file
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
XCEngine Test Runner
|
||||
|
||||
Unified test execution script for XCEngine test suites.
|
||||
Supports Windows and Linux platforms with CTest and Python wrapper integration.
|
||||
|
||||
Usage:
|
||||
python run_tests.py [options]
|
||||
|
||||
Options:
|
||||
--build Build tests before running
|
||||
--unit-only Run only unit tests (skip integration tests)
|
||||
--integration Run integration tests (includes unit tests)
|
||||
--ci CI mode (skip GUI-dependent integration tests)
|
||||
--dir <path> Run tests for specific directory
|
||||
--verbose,-v Verbose output
|
||||
--help,-h Show this help message
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
class Colors:
|
||||
RED = "\033[91m"
|
||||
GREEN = "\033[92m"
|
||||
YELLOW = "\033[93m"
|
||||
BLUE = "\033[94m"
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
|
||||
|
||||
class TestRunner:
|
||||
PLATFORM_WINDOWS = "Windows"
|
||||
PLATFORM_LINUX = "Linux"
|
||||
PLATFORM_MACOS = "Darwin"
|
||||
|
||||
def __init__(self, build_dir: Path, source_dir: Path, verbose: bool = False):
|
||||
self.build_dir = Path(build_dir)
|
||||
self.source_dir = Path(source_dir)
|
||||
self.verbose = verbose
|
||||
self.platform = self._detect_platform()
|
||||
self.results: List[Dict[str, Any]] = []
|
||||
self.total_passed = 0
|
||||
self.total_failed = 0
|
||||
self.total_time = 0.0
|
||||
|
||||
def _detect_platform(self) -> str:
|
||||
if sys.platform.startswith("win"):
|
||||
return self.PLATFORM_WINDOWS
|
||||
elif sys.platform.startswith("linux"):
|
||||
return self.PLATFORM_LINUX
|
||||
elif sys.platform.startswith("darwin"):
|
||||
return self.PLATFORM_MACOS
|
||||
return "Unknown"
|
||||
|
||||
def _log(self, message: str, color: str = ""):
|
||||
if color:
|
||||
print(f"{color}{message}{Colors.RESET}")
|
||||
else:
|
||||
print(message)
|
||||
|
||||
def _log_header(self, message: str):
|
||||
print()
|
||||
print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}")
|
||||
print(f"{Colors.BOLD}{Colors.BLUE}{message}{Colors.RESET}")
|
||||
print(f"{Colors.BOLD}{Colors.BLUE}{'=' * 60}{Colors.RESET}")
|
||||
|
||||
def _log_success(self, message: str):
|
||||
self._log(f"{Colors.GREEN}[PASS] {message}{Colors.RESET}")
|
||||
|
||||
def _log_error(self, message: str):
|
||||
self._log(f"{Colors.RED}[FAIL] {message}{Colors.RESET}")
|
||||
|
||||
def _log_warning(self, message: str):
|
||||
self._log(f"{Colors.YELLOW}[WARN] {message}{Colors.RESET}")
|
||||
|
||||
def _log_info(self, message: str):
|
||||
self._log(f" {message}")
|
||||
|
||||
def _run_command(
|
||||
self, cmd: List[str], cwd: Optional[Path] = None, capture_output: bool = True
|
||||
) -> subprocess.CompletedProcess:
|
||||
if self.verbose:
|
||||
self._log_info(f"Running: {' '.join(str(c) for c in cmd)}")
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, cwd=cwd, capture_output=capture_output, text=True, timeout=300
|
||||
)
|
||||
return result
|
||||
except subprocess.TimeoutExpired:
|
||||
self._log_error(f"Command timed out: {' '.join(str(c) for c in cmd)}")
|
||||
return subprocess.CompletedProcess(cmd, 1, "", "Timeout expired")
|
||||
except Exception as e:
|
||||
self._log_error(f"Command failed: {e}")
|
||||
return subprocess.CompletedProcess(cmd, 1, "", str(e))
|
||||
|
||||
def build_tests(self, config: str = "Debug") -> bool:
|
||||
self._log_header("Building Tests")
|
||||
|
||||
cmd = ["cmake", "--build", str(self.build_dir), "--config", config]
|
||||
|
||||
result = self._run_command(cmd)
|
||||
|
||||
if result.returncode == 0:
|
||||
self._log_success("Build completed successfully")
|
||||
return True
|
||||
else:
|
||||
self._log_error("Build failed")
|
||||
if self.verbose and result.stdout:
|
||||
print(result.stdout)
|
||||
return False
|
||||
|
||||
def _get_ctest_executable(self) -> str:
|
||||
if self.platform == self.PLATFORM_WINDOWS:
|
||||
return "ctest"
|
||||
return "ctest"
|
||||
|
||||
def _run_ctest_in_dir(self, test_dir: Path, label: str = "") -> Dict[str, Any]:
|
||||
ctest_exe = self._get_ctest_executable()
|
||||
|
||||
cmd = [ctest_exe, "-C", "Debug", "--output-on-failure"]
|
||||
|
||||
if self.verbose:
|
||||
cmd.append("--verbose")
|
||||
|
||||
start_time = time.time()
|
||||
result = self._run_command(cmd, cwd=test_dir)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
output = result.stdout + result.stderr
|
||||
|
||||
if "100% tests passed" in output or "passed" in output.lower():
|
||||
if "0 tests failed" in output:
|
||||
passed_match = output.find("tests passed")
|
||||
if passed_match != -1:
|
||||
try:
|
||||
num_str = (
|
||||
output[max(0, passed_match - 5) : passed_match]
|
||||
.strip()
|
||||
.split()[-1]
|
||||
)
|
||||
passed = int(num_str)
|
||||
except:
|
||||
passed = 1
|
||||
else:
|
||||
passed = 1
|
||||
|
||||
if "Failed" in output or result.returncode != 0:
|
||||
failed = 1
|
||||
|
||||
return {
|
||||
"name": label or f"Tests in {test_dir.name}",
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"time": elapsed,
|
||||
"returncode": result.returncode,
|
||||
"output": output if self.verbose else "",
|
||||
}
|
||||
|
||||
def run_ctest(self, test_dir: Optional[Path] = None) -> bool:
|
||||
if test_dir:
|
||||
test_dirs = [Path(test_dir)]
|
||||
else:
|
||||
test_dirs = [
|
||||
self.build_dir / "tests" / "math",
|
||||
self.build_dir / "tests" / "core",
|
||||
self.build_dir / "tests" / "containers",
|
||||
self.build_dir / "tests" / "memory",
|
||||
self.build_dir / "tests" / "threading",
|
||||
self.build_dir / "tests" / "debug",
|
||||
self.build_dir / "tests" / "Resources",
|
||||
self.build_dir / "tests" / "RHI" / "D3D12" / "unit",
|
||||
]
|
||||
|
||||
all_passed = True
|
||||
|
||||
for td in test_dirs:
|
||||
if not td.exists():
|
||||
self._log_warning(f"Test directory not found: {td}")
|
||||
continue
|
||||
|
||||
self._log_header(f"Running Tests: {td.name}")
|
||||
|
||||
result = self._run_ctest_in_dir(td, td.name)
|
||||
self.results.append(result)
|
||||
|
||||
if result["failed"] > 0:
|
||||
self._log_error(f"Failed: {result['name']}")
|
||||
all_passed = False
|
||||
else:
|
||||
self._log_success(f"Passed: {result['name']} ({result['time']:.2f}s)")
|
||||
|
||||
if self.verbose and result.get("output"):
|
||||
print(result["output"])
|
||||
|
||||
return all_passed
|
||||
|
||||
def _run_integration_test(
|
||||
self,
|
||||
test_name: str,
|
||||
exe_path: Path,
|
||||
output_ppm: str,
|
||||
gt_ppm: Path,
|
||||
threshold: int = 5,
|
||||
) -> Dict[str, Any]:
|
||||
wrapper_script = (
|
||||
self.source_dir
|
||||
/ "tests"
|
||||
/ "RHI"
|
||||
/ "D3D12"
|
||||
/ "integration"
|
||||
/ "run_integration_test.py"
|
||||
)
|
||||
|
||||
if not wrapper_script.exists():
|
||||
return {
|
||||
"name": test_name,
|
||||
"passed": 0,
|
||||
"failed": 1,
|
||||
"time": 0,
|
||||
"error": "Wrapper script not found",
|
||||
}
|
||||
|
||||
python_exe = sys.executable
|
||||
cmd = [
|
||||
python_exe,
|
||||
str(wrapper_script),
|
||||
str(exe_path),
|
||||
output_ppm,
|
||||
str(gt_ppm),
|
||||
str(threshold),
|
||||
]
|
||||
|
||||
start_time = time.time()
|
||||
result = self._run_command(cmd, cwd=exe_path.parent)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
passed = 1 if result.returncode == 0 else 0
|
||||
failed = 0 if passed else 1
|
||||
|
||||
return {
|
||||
"name": test_name,
|
||||
"passed": passed,
|
||||
"failed": failed,
|
||||
"time": elapsed,
|
||||
"returncode": result.returncode,
|
||||
"output": result.stdout + result.stderr,
|
||||
}
|
||||
|
||||
def run_integration_tests(self) -> bool:
|
||||
integration_dir = self.build_dir / "tests" / "RHI" / "D3D12" / "integration"
|
||||
integration_src_dir = (
|
||||
self.source_dir / "tests" / "RHI" / "D3D12" / "integration"
|
||||
)
|
||||
|
||||
if not integration_dir.exists():
|
||||
self._log_warning(
|
||||
f"Integration test directory not found: {integration_dir}"
|
||||
)
|
||||
return True
|
||||
|
||||
self._log_header("Running Integration Tests")
|
||||
|
||||
integration_tests = [
|
||||
("D3D12_Minimal_Integration", "minimal", "minimal.ppm"),
|
||||
("D3D12_Sphere_Integration", "sphere", "sphere.ppm"),
|
||||
("D3D12_Triangle_Integration", "triangle", "triangle.ppm"),
|
||||
("D3D12_Quad_Integration", "quad", "quad.ppm"),
|
||||
]
|
||||
|
||||
all_passed = True
|
||||
for test_name, test_folder, output_ppm in integration_tests:
|
||||
exe_path = (
|
||||
integration_dir
|
||||
/ test_folder
|
||||
/ "Debug"
|
||||
/ f"D3D12_{test_folder.title()}.exe"
|
||||
)
|
||||
gt_ppm = integration_src_dir / test_folder / "GT.ppm"
|
||||
|
||||
if exe_path.exists() and gt_ppm.exists():
|
||||
result = self._run_integration_test(
|
||||
test_name, exe_path, output_ppm, gt_ppm, 0
|
||||
)
|
||||
self.results.append(result)
|
||||
|
||||
if result["failed"] > 0:
|
||||
self._log_error(f"Failed: {result['name']}")
|
||||
if self.verbose:
|
||||
self._log_info(result.get("output", ""))
|
||||
all_passed = False
|
||||
else:
|
||||
self._log_success(
|
||||
f"Passed: {result['name']} ({result['time']:.2f}s)"
|
||||
)
|
||||
else:
|
||||
self._log_warning(f"{test_name} not found, skipping")
|
||||
|
||||
return all_passed
|
||||
|
||||
def _summarize_results(self):
|
||||
self._log_header("Test Summary")
|
||||
|
||||
total_passed = 0
|
||||
total_failed = 0
|
||||
total_time = 0.0
|
||||
|
||||
for result in self.results:
|
||||
total_passed += result.get("passed", 0)
|
||||
total_failed += result.get("failed", 0)
|
||||
total_time += result.get("time", 0)
|
||||
|
||||
self.total_passed = total_passed
|
||||
self.total_failed = total_failed
|
||||
self.total_time = total_time
|
||||
|
||||
print()
|
||||
print(f" Platform: {self.platform}")
|
||||
print(f" Total Passed: {Colors.GREEN}{total_passed}{Colors.RESET}")
|
||||
print(
|
||||
f" Total Failed: {Colors.RED if total_failed > 0 else Colors.GREEN}{total_failed}{Colors.RESET}"
|
||||
)
|
||||
print(f" Total Time: {total_time:.2f}s")
|
||||
print()
|
||||
|
||||
def run(
|
||||
self,
|
||||
build: bool = False,
|
||||
unit_only: bool = False,
|
||||
integration: bool = False,
|
||||
ci_mode: bool = False,
|
||||
test_dir: Optional[Path] = None,
|
||||
) -> int:
|
||||
|
||||
print()
|
||||
print(f"{Colors.BOLD}XCEngine Test Runner{Colors.RESET}")
|
||||
print(f" Platform: {self.platform}")
|
||||
print(f" Build Dir: {self.build_dir}")
|
||||
print(f" Source Dir: {self.source_dir}")
|
||||
|
||||
if build:
|
||||
if not self.build_tests():
|
||||
self._log_error("Build failed, aborting tests")
|
||||
return 1
|
||||
|
||||
if test_dir:
|
||||
self._log_info(f"Running tests for: {test_dir}")
|
||||
if self._run_ctest_in_dir(Path(test_dir)).get("failed", 0) > 0:
|
||||
self._summarize_results()
|
||||
return 1
|
||||
self._summarize_results()
|
||||
return 0
|
||||
|
||||
if not self.run_ctest():
|
||||
self._summarize_results()
|
||||
return 1
|
||||
|
||||
if integration or (not unit_only and not ci_mode):
|
||||
if not self.run_integration_tests():
|
||||
self._summarize_results()
|
||||
return 1
|
||||
|
||||
self._summarize_results()
|
||||
|
||||
if self.total_failed > 0:
|
||||
self._log_error("Some tests failed")
|
||||
return 1
|
||||
else:
|
||||
self._log_success("All tests passed!")
|
||||
return 0
|
||||
|
||||
|
||||
def find_build_dir() -> Optional[Path]:
|
||||
possible_dirs = [
|
||||
Path(__file__).parent.parent / "build",
|
||||
Path.cwd() / "build",
|
||||
Path.cwd(),
|
||||
]
|
||||
|
||||
for d in possible_dirs:
|
||||
if d.exists() and (d / "CMakeCache.txt").exists():
|
||||
return d
|
||||
|
||||
script_dir = Path(__file__).parent.parent
|
||||
test_build = script_dir / "tests" / "build"
|
||||
if test_build.exists():
|
||||
return test_build
|
||||
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def find_source_dir() -> Path:
|
||||
return Path(__file__).parent.parent
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="XCEngine Test Runner - Unified test execution script",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python run_tests.py Run all tests
|
||||
python run_tests.py --build Build and run tests
|
||||
python run_tests.py --unit-only Run only unit tests
|
||||
python run_tests.py --ci CI mode (skip GUI integration tests)
|
||||
python run_tests.py --dir tests/math Run specific test directory
|
||||
python run_tests.py --verbose Verbose output
|
||||
""",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--build", "-b", action="store_true", help="Build tests before running"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--unit-only",
|
||||
"-u",
|
||||
action="store_true",
|
||||
help="Run only unit tests (skip integration tests)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--integration",
|
||||
"-i",
|
||||
action="store_true",
|
||||
help="Run integration tests (includes unit tests)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--ci",
|
||||
"-c",
|
||||
action="store_true",
|
||||
help="CI mode (skip GUI-dependent integration tests)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--dir", "-d", type=Path, help="Run tests for specific directory"
|
||||
)
|
||||
|
||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
|
||||
|
||||
parser.add_argument(
|
||||
"--build-dir", type=Path, help="Build directory (default: auto-detect)"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--source-dir", type=Path, help="Source directory (default: auto-detect)"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
build_dir = args.build_dir if args.build_dir else find_build_dir()
|
||||
source_dir = args.source_dir if args.source_dir else find_source_dir()
|
||||
|
||||
if not build_dir:
|
||||
print(f"{Colors.RED}Error: Could not find build directory{Colors.RESET}")
|
||||
print("Please specify --build-dir or run from a directory with CMake cache")
|
||||
return 1
|
||||
|
||||
runner = TestRunner(build_dir, source_dir, verbose=args.verbose)
|
||||
|
||||
return runner.run(
|
||||
build=args.build,
|
||||
unit_only=args.unit_only,
|
||||
integration=args.integration,
|
||||
ci_mode=args.ci,
|
||||
test_dir=args.dir,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main() or 0)
|
||||
Reference in New Issue
Block a user