diff --git a/.github/workflows/build-and-test-callable.yaml b/.github/workflows/build-and-test-callable.yaml index 15306c7a4..8cdfad3f5 100644 --- a/.github/workflows/build-and-test-callable.yaml +++ b/.github/workflows/build-and-test-callable.yaml @@ -11,6 +11,11 @@ on: required: false default: 'main' type: string + OffloadTest-fork: + description: 'Test Suite fork' + required: false + default: 'llvm' + type: string LLVM-branch: description: 'LLVM Branch' required: false @@ -71,6 +76,11 @@ on: required: false default: 'main' type: string + OffloadTest-fork: + description: 'Test Suite Fork' + required: false + default: 'llvm' + type: string LLVM-branch: description: 'LLVM Branch' required: false @@ -147,13 +157,12 @@ jobs: - name: Checkout OffloadTest uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: - repository: llvm/offload-test-suite + repository: ${{ inputs.OffloadTest-fork }}/offload-test-suite ref: ${{ inputs.OffloadTest-branch }} path: OffloadTest fetch-depth: 1 - name: Checkout Golden Images uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - if: inputs.SplitBuild != true with: repository: llvm/offload-golden-images ref: main @@ -190,9 +199,9 @@ jobs: cd llvm-project mkdir build cd build - cmake -G Ninja ${{ inputs.LLVM-ExtraCMakeArgs }} -DCMAKE_BUILD_TYPE=${{ inputs.BuildType }} -DLLVM_ENABLE_ASSERTIONS=On -C ${{ github.workspace }}/llvm-project/clang/cmake/caches/HLSL.cmake -C ${{ github.workspace }}/OffloadTest/cmake/caches/sccache.cmake -DDXC_DIR=${{ github.workspace }}/DXC/build/bin -DLLVM_EXTERNAL_OFFLOADTEST_SOURCE_DIR=${{ github.workspace }}/OffloadTest -DLLVM_EXTERNAL_PROJECTS="OffloadTest" -DLLVM_LIT_ARGS="--xunit-xml-output=testresults.xunit.xml -v" -DOFFLOADTEST_TEST_CLANG=${{steps.Test-Clang.outputs.TEST_CLANG || 'Off' }} -DGOLDENIMAGE_DIR=${{ github.workspace }}/golden-images ${{ github.workspace }}/llvm-project/llvm/ + cmake -G Ninja ${{ inputs.LLVM-ExtraCMakeArgs }} -DCMAKE_BUILD_TYPE=${{ inputs.BuildType }} -DLLVM_ENABLE_ASSERTIONS=On -DLLVM_EXTERNAL_PROJECTS="OffloadTest" -DHLSL_ENABLE_OFFLOAD_DISTRIBUTION=${{ inputs.SplitBuild == true && 'On' || 'Off' }} -C ${{ github.workspace }}/llvm-project/clang/cmake/caches/HLSL.cmake -C ${{ github.workspace }}/OffloadTest/cmake/caches/sccache.cmake -DDXC_DIR=${{ github.workspace }}/DXC/build/bin -DLLVM_EXTERNAL_OFFLOADTEST_SOURCE_DIR=${{ github.workspace }}/OffloadTest -DLLVM_LIT_ARGS="--xunit-xml-output=testresults.xunit.xml -v" -DOFFLOADTEST_TEST_CLANG=${{steps.Test-Clang.outputs.TEST_CLANG || 'Off' }} -DGOLDENIMAGE_DIR=${{ github.workspace }}/golden-images -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ github.workspace }}/llvm-project/llvm/ ninja hlsl-test-depends - - name: Dump GPU Info + - name: Dump GPU Info (build runner) if: inputs.SplitBuild != true run: | cd llvm-project @@ -232,19 +241,69 @@ jobs: name: dxdiag-${{ inputs.SKU }}-${{inputs.TestTarget}}.txt path: ${{ runner.temp }}/dxdiag.txt - # When SplitBuild is true, package and upload build - # artifacts for the test job. - - name: Package build artifacts + # When SplitBuild is true, install the LLVM/DXC distribution and the + # standalone offload test suite into a single portable prefix, then + # tar that prefix for the test job. The prefix is fully self-contained: + # tools under bin/, test sources + configure script under + # share/hlsl-test-suite/. No CMake, ninja, or compiler toolchain is + # required on the test runner. + - name: Install distribution + if: inputs.SplitBuild == true + shell: bash + run: | + set -euxo pipefail + cd $GITHUB_WORKSPACE/llvm-project/build + ninja install-distribution install-offload-tools install-offload-test-suite + # Stage DXC into a portable bin/+lib/ tree at dxc-dist. + # See docs/offload-distribution.md ("DXC prefix") for the + # rationale (DXC's install targets don't cover everything we + # need, so we cherry-pick from the build directory). + buildbin=$GITHUB_WORKSPACE/DXC/build/bin + buildlib=$GITHUB_WORKSPACE/DXC/build/lib + distbin=$GITHUB_WORKSPACE/dxc-dist/bin + distlib=$GITHUB_WORKSPACE/dxc-dist/lib + mkdir -p "$distbin" "$distlib" + case "$RUNNER_OS" in + Windows) + cp "$buildbin/dxc.exe" "$distbin/" + cp "$buildbin/dxv.exe" "$distbin/" + cp "$buildbin/dxcompiler.dll" "$distbin/" + cp "$buildbin/dxil.dll" "$distbin/" + # PDBs only exist for configs that emit /Zi (Debug, + # RelWithDebInfo). Release / MinSizeRel produce no + # .pdb files, so we skip the copy in those cases. + case "${{ inputs.BuildType }}" in + Debug|RelWithDebInfo) + cp "$buildbin/dxc.pdb" "$distbin/" + cp "$buildbin/dxv.pdb" "$distbin/" + cp "$buildbin/dxcompiler.pdb" "$distbin/" + cp "$buildbin/dxil.pdb" "$distbin/" ;; + esac + cp "$buildlib/dxcompiler.lib" "$distlib/" + cp "$buildlib/dxil.lib" "$distlib/" ;; + Linux) + cp "$buildbin/dxc" "$distbin/" + cp "$buildbin/dxv" "$distbin/" + cp "$buildlib/libdxcompiler.so" "$distlib/" + cp "$buildlib/libdxil.so" "$distlib/" ;; + macOS) + cp "$buildbin/dxc" "$distbin/" + cp "$buildbin/dxv" "$distbin/" + cp "$buildlib/libdxcompiler.dylib" "$distlib/" + cp "$buildlib/libdxil.dylib" "$distlib/" ;; + *) + echo "Unsupported RUNNER_OS=$RUNNER_OS" >&2; exit 1 ;; + esac + - name: Package install prefix if: inputs.SplitBuild == true shell: bash run: | cd $GITHUB_WORKSPACE - tar cf $RUNNER_TEMP/build-artifacts.tar \ - --exclude='*.obj' \ - --exclude='*.o' \ - --exclude='*.ilk' \ - --exclude='*.pdb' \ - DXC/build/bin llvm-project/build + # Use stdout redirection so tar doesn't try to parse the + # destination Windows drive letter (e.g. C:\...) as a remote + # host:path tuple. + tar cf - -C install . > "$RUNNER_TEMP/build-artifacts.tar" + tar cf - -C dxc-dist . > "$RUNNER_TEMP/dxc-artifacts.tar" - name: Upload build artifacts if: inputs.SplitBuild == true uses: actions/upload-artifact@v4 @@ -252,6 +311,13 @@ jobs: name: build-${{ inputs.SKU }}-${{ inputs.TestTarget }} path: ${{ runner.temp }}/build-artifacts.tar retention-days: 1 + - name: Upload DXC artifacts + if: inputs.SplitBuild == true + uses: actions/upload-artifact@v4 + with: + name: dxc-${{ inputs.SKU }}-${{ inputs.TestTarget }} + path: ${{ runner.temp }}/dxc-artifacts.tar + retention-days: 1 test: if: inputs.SplitBuild == true @@ -260,53 +326,30 @@ jobs: checks: write runs-on: [self-hosted, "hlsl-${{ inputs.SKU }}"] steps: - - name: Checkout DXC - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: Microsoft/DirectXShaderCompiler - ref: ${{ inputs.DXC-branch }} - path: DXC - fetch-depth: 1 - submodules: true - - name: Checkout LLVM - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: ${{ inputs.LLVM-fork }}/llvm-project - ref: ${{ inputs.LLVM-branch }} - path: llvm-project - fetch-depth: 1 - - name: Checkout OffloadTest - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: llvm/offload-test-suite - ref: ${{ inputs.OffloadTest-branch }} - path: OffloadTest - fetch-depth: 1 - - name: Checkout Golden Images - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - with: - repository: llvm/offload-golden-images - ref: main - path: golden-images - fetch-depth: 1 + # Test job consumes ONLY the install tarball produced by the build job. + # No source checkouts of llvm-project / DXC / OffloadTest are needed + # here -- everything required to run lit lives inside the prefix. - name: Download build artifacts uses: actions/download-artifact@v4 with: name: build-${{ inputs.SKU }}-${{ inputs.TestTarget }} path: ${{ runner.temp }} - - name: Extract build artifacts + - name: Download DXC artifacts + uses: actions/download-artifact@v4 + with: + name: dxc-${{ inputs.SKU }}-${{ inputs.TestTarget }} + path: ${{ runner.temp }} + - name: Extract install prefix shell: bash run: | + set -euxo pipefail cd $GITHUB_WORKSPACE - tar xf $RUNNER_TEMP/build-artifacts.tar - - name: Refresh artifact timestamps - shell: python - run: | - import pathlib - for d in ['DXC/build/bin', 'llvm-project/build']: - for f in pathlib.Path(d).rglob('*'): - if f.is_file(): - f.touch() + mkdir -p install dxc-dist + # Pipe the archive in via stdin and use a relative -C path so + # tar doesn't try to parse the Windows drive letter or treat + # backslashes as escape characters. + tar xf - -C install < "$RUNNER_TEMP/build-artifacts.tar" + tar xf - -C dxc-dist < "$RUNNER_TEMP/dxc-artifacts.tar" - name: Setup Windows x64 if: inputs.OS == 'windows' && runner.arch != 'ARM64' uses: llvm/actions/setup-windows@89a8cf80982d830faab019237860b344a6390c30 # main @@ -320,24 +363,60 @@ jobs: - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: - pip-install: -r ${{ github.workspace }}/OffloadTest/test/requirements.txt - - name: Dump GPU Info + # The test runner needs only lit + pyyaml; no source checkout is + # available so we can't pip install -r requirements.txt. Keep + # this list in sync with OffloadTest/test/requirements.txt. + pip-install: lit pyyaml + - name: Dump GPU Info (test runner) + shell: bash run: | - cd llvm-project - cd build - ./bin/api-query + $GITHUB_WORKSPACE/install/bin/api-query + - name: Resolve lit suite from TestTarget + id: suite + shell: bash + run: | + # SplitBuild callers always pass a single specific suite as + # TestTarget=check-hlsl-. Strip the prefix to get the + # suite name configure-test-suite.py understands. + set -euxo pipefail + target='${{ inputs.TestTarget }}' + case "$target" in + check-hlsl-*) + suite="${target#check-hlsl-}" + ;; + *) + echo "::error::SplitBuild requires TestTarget=check-hlsl-, got: $target" + exit 1 + ;; + esac + echo "suite=$suite" >> $GITHUB_OUTPUT + echo "Will run lit suite: $suite" + - name: Configure standalone test suite + shell: bash + run: | + set -euxo pipefail + cd "$GITHUB_WORKSPACE/install/share/hlsl-test-suite" + # Locate dxc in the separate DXC install prefix. .exe on Windows, + # bare name elsewhere. + dxc_path="$GITHUB_WORKSPACE/dxc-dist/bin/dxc" + if [ -x "${dxc_path}.exe" ]; then dxc_path="${dxc_path}.exe"; fi + python configure-test-suite.py \ + --suite "${{ steps.suite.outputs.suite }}" \ + --dxc-path "$dxc_path" - name: Run HLSL Tests + shell: bash run: | - cd llvm-project - cd build - ninja check-hlsl-unit - ninja ${{ inputs.TestTarget }} + set -euxo pipefail + cd $GITHUB_WORKSPACE/install/share/hlsl-test-suite + suite='${{ steps.suite.outputs.suite }}' + lit -v "run/test/$suite" \ + --xunit-xml-output="run/test/$suite/testresults.xunit.xml" - name: Publish Test Results uses: EnricoMi/publish-unit-test-result-action/macos@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0 if: always() && inputs.OS == 'macOS' with: comment_mode: off - files: llvm-project/build/**/testresults.xunit.xml + files: ${{ github.workspace }}/install/share/hlsl-test-suite/run/test/**/testresults.xunit.xml - name: Run dxdiag (Windows only) if: inputs.OS == 'windows' && failure() shell: powershell diff --git a/CMakeLists.txt b/CMakeLists.txt index c7bbbf770..144c074f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,13 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include macro(add_offloadtest_tool name) add_llvm_executable(${name} ${ARGN}) set_target_properties(${name} PROPERTIES FOLDER "HLSL Test/Tools") + # Install offload-test tools under a single umbrella component so the + # standalone test distribution can pull them all with one + # `install-offload-tools` target. LLVM_TOOLS_INSTALL_DIR defaults to + # `bin` when building inside an LLVM tree. + install(TARGETS ${name} + COMPONENT offload-tools + RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") endmacro() macro(add_offloadtest_library name) @@ -132,3 +139,13 @@ add_subdirectory(tools) add_subdirectory(unittests) add_subdirectory(test) + +# Aggregate install target for the offload-tools component. After the +# `add_subdirectory(tools)` call above has registered every tool under the +# `offload-tools` component via `add_offloadtest_tool`, this provides a +# single `install-offload-tools` target that builds + installs all of them. +if (NOT LLVM_ENABLE_IDE) + add_llvm_install_targets(install-offload-tools + COMPONENT offload-tools + DEPENDS api-query imgdiff offloader) +endif() diff --git a/docs/offload-distribution.md b/docs/offload-distribution.md new file mode 100644 index 000000000..6c30cd714 --- /dev/null +++ b/docs/offload-distribution.md @@ -0,0 +1,179 @@ +# Offload Test Suite Distribution + +The offload test suite supports a *split build / test* mode where binaries +and test data are built on one machine, packaged as portable install +prefixes, and consumed on a different machine to run the lit tests. This +is the model used by the `SplitBuild=true` path of the +`build-and-test-callable.yaml` workflow. + +This document describes the install layout, prerequisites, and how to +configure and run the suite from an installed prefix. + +## Components + +A complete deployment consists of two install prefixes: + +1. **LLVM + OffloadTest prefix** — produced by the `install-distribution`, + `install-offload-tools`, and `install-offload-test-suite` targets. Built + with `clang/cmake/caches/HLSL.cmake` and `HLSL_ENABLE_OFFLOAD_DISTRIBUTION=On`. + Contains: + - `bin/` — clang, FileCheck, split-file, not, obj2yaml, api-query, + offloader, and the other tools created by `add_offloadtest_tool`. + - `include/`, `lib/clang//include/` — clang resource headers + (`hlsl-resource-headers` component). + - `share/hlsl-test-suite/` — test sources, `lit.site.cfg.py.in` template, + `configure-test-suite.py`, and golden images. + +2. **DXC prefix** — a hand-curated `bin/` + `lib/` tree containing just + the DXC binaries the test runner needs. Kept separate from the LLVM + prefix to avoid header / binary conflicts between Clang's HLSL-related + headers and DXC's. Contents: + + Windows: + - `bin/dxc.exe`, `bin/dxv.exe` — DXC compiler and validator. + - `bin/dxcompiler.dll` — DXC's compiler shared library (placed + next to the executables so Windows' DLL search finds it via the + app-directory rule, no `PATH` munging required). + - `bin/dxil.dll` — DXC's signing / validation library. + - `bin/dxc.pdb`, `bin/dxv.pdb`, `bin/dxcompiler.pdb`, `bin/dxil.pdb` — + debug symbol files (PDBs) shipped alongside their corresponding + binaries so crash dumps and live debugger sessions can resolve + symbols. The Windows debugger looks for `.pdb` files next to the + `.exe` / `.dll`, so no symbol-server configuration is required. + PDBs are shipped only when the build config produces them (any + config that enables `/Zi`, e.g. `Debug` or `RelWithDebInfo`); a + pure `Release` build omits them and the distribution simply has + no PDBs. + - `lib/dxcompiler.lib`, `lib/dxil.lib` — Windows import libraries + for downstream consumers that link against the DLLs. + + Linux / macOS: + - `bin/dxc`, `bin/dxv`. + - `lib/libdxcompiler.{so,dylib}`, `lib/libdxil.{so,dylib}` — the + binaries have RUNPATH set to find them via `../lib`. + + We don't ship DXC's `cmake --install` output. A top-level `ninja + install` walks every `cmake_install.cmake`, including LLVM tools + (e.g. `llvm-as`) that aren't built by the default `ninja` target, so + it fails. The per-component install targets (`install-dxc`, + `install-dxcompiler`) work but only cover a subset of the files we + need: `dxv` has no `install-dxv` custom target, `dxil` has no install + rule at all (it's a prebuilt signing library bundled with DXC source), + and the Windows import libraries (`.lib`) aren't installed either. + Instead we copy everything we need straight out of the build + directory's `bin/` and `lib/`. + +## Building + +From a configured llvm-project build tree that has OffloadTest enabled as +an external project: + +``` +cmake -G Ninja \ + -DHLSL_ENABLE_OFFLOAD_DISTRIBUTION=On \ + -DLLVM_EXTERNAL_PROJECTS=OffloadTest \ + -C llvm-project/clang/cmake/caches/HLSL.cmake \ + -DLLVM_EXTERNAL_OFFLOADTEST_SOURCE_DIR= \ + -DCMAKE_INSTALL_PREFIX= \ + \ + llvm-project/llvm + +ninja install-distribution install-offload-tools install-offload-test-suite + +# See "DXC prefix" above for why we copy from the build folder instead +# of using `cmake --install` or per-component install targets. +``` + +Then assemble the DXC prefix by copying the relevant files out of +`` into a clean `` tree, e.g. on Windows: + +``` +/bin/dxc.exe -> /bin/ +/bin/dxv.exe -> /bin/ +/bin/dxcompiler.dll -> /bin/ +/bin/dxil.dll -> /bin/ +/bin/dxc.pdb -> /bin/ +/bin/dxv.pdb -> /bin/ +/bin/dxcompiler.pdb -> /bin/ +/bin/dxil.pdb -> /bin/ +/lib/dxcompiler.lib -> /lib/ +/lib/dxil.lib -> /lib/ +``` + +The HLSL.cmake cache file enforces that `OffloadTest` is in +`LLVM_EXTERNAL_PROJECTS` when `HLSL_ENABLE_OFFLOAD_DISTRIBUTION` is on; the +configure step will fail fast otherwise. Note that +`-DHLSL_ENABLE_OFFLOAD_DISTRIBUTION=On` and `-DLLVM_EXTERNAL_PROJECTS=OffloadTest` +must appear on the `cmake` command line *before* the `-C HLSL.cmake` argument +so they're set in the cache before the cache script runs. + +## Packaging + +Tar each prefix and ship the tarballs to the test runner: + +``` +tar cf llvm-prefix.tar -C . +tar cf dxc-prefix.tar -C . +``` + +## Test runner prerequisites + +- Python 3.6+ +- `pip install lit pyyaml` +- GPU drivers appropriate for the suite (D3D12 / Vulkan / Metal). +- The two extracted prefixes from the build runner. + +No CMake, ninja, or compiler toolchain is required on the test runner. + +## Configuring and running + +Extract both prefixes, then run the configure script with `--dxc-path` +pointing at the DXC binary in its prefix: + +``` +mkdir install dxc-dist +tar xf llvm-prefix.tar -C install +tar xf dxc-prefix.tar -C dxc-dist + +cd install/share/hlsl-test-suite +python configure-test-suite.py \ + --suite clang-d3d12 \ + --dxc-path ../../../dxc-dist/bin/dxc[.exe] +``` + +This emits a fully-substituted `run/test//lit.site.cfg.py`. + +`dxc[.exe]` finds its runtime libraries automatically: on Windows the +DLLs sit next to the executable in `bin/`, and on Linux/macOS the +binaries have RUNPATH set to locate `../lib`. No `PATH` / +`LD_LIBRARY_PATH` / `DYLD_LIBRARY_PATH` setup is required. + +Then run lit: + +``` +lit -v run/test/clang-d3d12 +``` + +`configure-test-suite.py --list-suites` prints the available suite names. + +## Suites + +| Name | Backend | Compiler | +|-------------------|-------------|----------| +| `d3d12` | DirectX 12 | DXC | +| `vk` | Vulkan | DXC | +| `mtl` | Metal | DXC | +| `warp-d3d12` | WARP (D3D12)| DXC | +| `clang-d3d12` | DirectX 12 | Clang | +| `clang-vk` | Vulkan | Clang | +| `clang-mtl` | Metal | Clang | +| `clang-warp-d3d12`| WARP (D3D12)| Clang | + +Clang suites do not require DXC; you can omit `--dxc-path` for those. + +## CI usage + +The reusable workflow `.github/workflows/build-and-test-callable.yaml` +implements this flow when invoked with `SplitBuild=true`. The build job +produces two artifacts (`build--` and `dxc--`) +and the test job consumes both. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6467681ab..cfa535b74 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -115,6 +115,12 @@ endforeach() # expanding it into lit. cmake_path(SET DXC_DIR NORMALIZE "${DXC_DIR}") +# The lit.site.cfg.py.in templates expect @OFFLOADTEST_TEST_ROOT@ — the +# directory containing test/lit.cfg.py. In an in-tree build that's the +# project source dir; in the standalone install layout it's the install +# share dir (set by configure-test-suite.py). +set(OFFLOADTEST_TEST_ROOT ${OFFLOADTEST_SOURCE_DIR}) + function(add_offloadtest_lit_suite suite) configure_lit_site_cfg( ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in @@ -191,3 +197,50 @@ endif() umbrella_lit_testsuite_end(check-hlsl) set_target_properties(check-hlsl PROPERTIES FOLDER "HLSL tests") + +# === Standalone test suite distribution === +# +# See docs/offload-distribution.md for an overview. All test infrastructure +# is installed under a single `offload-test-suite` component. Build with +# `ninja install-offload-test-suite`. +set(OFFLOADTEST_INSTALL_TESTDIR "share/hlsl-test-suite" CACHE STRING + "Install directory (relative to CMAKE_INSTALL_PREFIX) for the standalone offload test suite distribution.") + +# Install the test source tree. Using two separate install(DIRECTORY) calls +# keeps EXCLUDE patterns out of FILES_MATCHING, where EXCLUDE inside +# FILES_MATCHING is interpreted as an include-override. +install(DIRECTORY ${OFFLOADTEST_SOURCE_DIR}/test/ + DESTINATION ${OFFLOADTEST_INSTALL_TESTDIR}/test + COMPONENT offload-test-suite + PATTERN "Output" EXCLUDE + PATTERN "__pycache__" EXCLUDE + PATTERN "CMakeLists.txt" EXCLUDE + PATTERN "*.py.in" EXCLUDE) + +# Install the canonical lit.site.cfg.py template so configure-test-suite.py +# can substitute @VAR@ tokens against the same template the in-tree build +# uses, avoiding any drift between the two code paths. +install(FILES ${OFFLOADTEST_SOURCE_DIR}/test/lit.site.cfg.py.in + DESTINATION ${OFFLOADTEST_INSTALL_TESTDIR} + COMPONENT offload-test-suite) + +# Install the standalone configure script. +install(PROGRAMS ${OFFLOADTEST_SOURCE_DIR}/utils/configure-test-suite.py + DESTINATION ${OFFLOADTEST_INSTALL_TESTDIR} + COMPONENT offload-test-suite) + +# If golden images were available at configure time, ship them inside the +# distribution so the test runner doesn't need to re-checkout the repo. +if (GOLDENIMAGE_DIR AND EXISTS "${GOLDENIMAGE_DIR}") + file(TO_CMAKE_PATH "${GOLDENIMAGE_DIR}" _goldenimage_dir) + install(DIRECTORY "${_goldenimage_dir}/" + DESTINATION ${OFFLOADTEST_INSTALL_TESTDIR}/golden-images + COMPONENT offload-test-suite + PATTERN ".git" EXCLUDE + PATTERN ".gitattributes" EXCLUDE) +endif() + +if (NOT LLVM_ENABLE_IDE) + add_llvm_install_targets(install-offload-test-suite + COMPONENT offload-test-suite) +endif() diff --git a/test/Unit/lit.site.cfg.py.in b/test/Unit/lit.site.cfg.py.in index c47da732a..372162a76 100644 --- a/test/Unit/lit.site.cfg.py.in +++ b/test/Unit/lit.site.cfg.py.in @@ -7,7 +7,7 @@ import subprocess import lit.formats config.offloadtest_obj_root = path(r"@OFFLOADTEST_BINARY_DIR@") -config.offloadtest_src_root = path(r"@OFFLOADTEST_SOURCE_DIR@") +config.offloadtest_src_root = path(r"@OFFLOADTEST_TEST_ROOT@") config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@") config.gtest_run_under = lit_config.substitute(r"@LLVM_GTEST_RUN_UNDER@") diff --git a/test/lit.site.cfg.py.in b/test/lit.site.cfg.py.in index 2bafae667..e99663e07 100644 --- a/test/lit.site.cfg.py.in +++ b/test/lit.site.cfg.py.in @@ -3,7 +3,7 @@ import sys config.offloadtest_obj_root = path(r"@OFFLOADTEST_BINARY_DIR@") -config.offloadtest_src_root = path(r"@OFFLOADTEST_SOURCE_DIR@") +config.offloadtest_src_root = path(r"@OFFLOADTEST_TEST_ROOT@") config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@")) config.offloadtest_dxc = '"' + path(r"@DXC_EXECUTABLE@") + '"' config.offloadtest_supports_spirv = @SUPPORTS_SPIRV@ diff --git a/utils/configure-test-suite.py b/utils/configure-test-suite.py new file mode 100644 index 000000000..4d233466f --- /dev/null +++ b/utils/configure-test-suite.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Configure the HLSL offload test suite for standalone execution. + +Generates lit.site.cfg.py files for running the offload test suite from an +installed distribution, outside any LLVM/CMake build tree. Reads the +canonical lit.site.cfg.py.in template installed alongside this script and +substitutes @VAR@ tokens with values appropriate for *this* (test) machine. + +Prerequisites on the target machine: + * Python 3.6+ + * pip install lit pyyaml + * GPU drivers (D3D12, Vulkan, or Metal depending on suite) + * For non-clang suites: a DXC executable on PATH or via --dxc-path + +Typical install layout this script expects: + /bin/ tool binaries (offloader, FileCheck, ...) + /share/hlsl-test-suite/ this script + lit.site.cfg.py.in + test/ + +Usage: + ./configure-test-suite.py --suite clang-d3d12 + lit run/test/clang-d3d12 -v +""" + +import argparse +import os +import pathlib +import platform +import re +import shutil +import sys + +# Suite name -> (d3d12, vk, mtl, clang, warp, description). +SUITES = { + "d3d12": (True, False, False, False, False, "DXC-compiled shaders on DirectX 12"), + "vk": (False, True, False, False, False, "DXC-compiled shaders on Vulkan"), + "mtl": (False, False, True, False, False, "DXC-compiled shaders on Metal"), + "clang-d3d12": ( + True, + False, + False, + True, + False, + "Clang-compiled shaders on DirectX 12", + ), + "clang-vk": (False, True, False, True, False, "Clang-compiled shaders on Vulkan"), + "clang-mtl": (False, False, True, True, False, "Clang-compiled shaders on Metal"), + "warp-d3d12": ( + True, + False, + False, + False, + True, + "DXC-compiled shaders on WARP (Windows)", + ), + "clang-warp-d3d12": ( + True, + False, + False, + True, + True, + "Clang-compiled shaders on WARP (Windows)", + ), +} + +# Self-contained replacement for @LIT_SITE_CFG_IN_HEADER@. The in-tree +# header (provided by LLVM's configure_lit_site_cfg) wraps paths with a +# helper that rewrites build-tree absolutes; here every path we substitute +# is already absolute and correct for the test runner, so `path()` is just +# an identity function. +HEADER = """\ +# Autogenerated by configure-test-suite.py +# Do not edit - regenerate with configure-test-suite.py. +import os +import platform +def path(p): + return p +""" + + +def emit_site_cfg(template_text, substitutions): + """Replace @VAR@ tokens in template_text. Unsubstituted @VAR@ tokens + produce a hard error so we don't silently ship a half-configured file.""" + text = template_text.replace("@LIT_SITE_CFG_IN_HEADER@", HEADER) + for key, value in substitutions.items(): + text = text.replace("@{}@".format(key), value) + # Catch any leftover @TOKEN@ that we didn't account for. + leftover = re.findall(r"@[A-Za-z_][A-Za-z0-9_]*@", text) + if leftover: + raise SystemExit( + "configure-test-suite.py: unsubstituted tokens in template: " + + ", ".join(sorted(set(leftover))) + ) + return text + + +def main(): + script_dir = pathlib.Path(__file__).resolve().parent + default_prefix = ( + script_dir.parent.parent + ) # /share/hlsl-test-suite -> + + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--suite", + action="append", + dest="suites", + metavar="NAME", + help="Test suite to configure (repeatable).", + ) + p.add_argument( + "--list-suites", action="store_true", help="List available suites and exit." + ) + p.add_argument( + "--prefix", + type=pathlib.Path, + default=default_prefix, + help="Install prefix containing bin/ (default: auto-detected from script location).", + ) + p.add_argument( + "--bin-dir", + type=pathlib.Path, + default=None, + help="Override directory containing tool binaries (default: /bin).", + ) + p.add_argument( + "--output-dir", + type=pathlib.Path, + default=None, + help="Output directory for generated configs (default: /run).", + ) + p.add_argument( + "--dxc-path", + type=pathlib.Path, + default=None, + help="Path to dxc executable (for non-clang suites; default: from PATH).", + ) + p.add_argument( + "--golden-images", + type=pathlib.Path, + default=None, + help="Override golden-images directory (default: /golden-images if present).", + ) + p.add_argument( + "--enable-debug", + dest="enable_debug", + action="store_true", + default=True, + help="Enable runtime debug layers (default: on).", + ) + p.add_argument( + "--no-debug", + dest="enable_debug", + action="store_false", + help="Disable runtime debug layers.", + ) + p.add_argument( + "--enable-validation", + action="store_true", + default=False, + help="Enable runtime validation layers (default: off).", + ) + p.add_argument( + "--os-name", + default=platform.system(), + help="Override CMAKE_SYSTEM_NAME-equivalent OS name (default: auto-detect).", + ) + p.add_argument( + "--warp-arch", + default="", + help="WARP_ARCHITECTURE value (only for *-warp-d3d12 suites).", + ) + + args = p.parse_args() + + if args.list_suites: + print("Available test suites:") + for name in sorted(SUITES): + print(" {:<18s} {}".format(name, SUITES[name][5])) + return 0 + + if not args.suites: + p.error("--suite is required (use --list-suites to see options)") + + unknown = [s for s in args.suites if s not in SUITES] + if unknown: + p.error( + "unknown suite(s): {} (available: {})".format( + ", ".join(unknown), ", ".join(sorted(SUITES)) + ) + ) + + bin_dir = (args.bin_dir or (args.prefix / "bin")).resolve() + if not bin_dir.is_dir(): + p.error("bin dir does not exist: {}".format(bin_dir)) + + template_path = script_dir / "lit.site.cfg.py.in" + if not template_path.is_file(): + p.error("missing template alongside script: {}".format(template_path)) + template_text = template_path.read_text(encoding="utf-8") + + output_base = (args.output_dir or (script_dir / "run")).resolve() + test_src_root = (script_dir / "test").resolve() + if not test_src_root.is_dir(): + p.error("missing installed test sources: {}".format(test_src_root)) + + # Resolve DXC if any non-clang suite is being configured. + needs_dxc = any(not SUITES[s][3] for s in args.suites) + dxc_path = args.dxc_path + if needs_dxc and dxc_path is None: + found = shutil.which("dxc") or shutil.which("dxc.exe") + if found: + dxc_path = pathlib.Path(found).resolve() + if needs_dxc and (dxc_path is None or not dxc_path.is_file()): + print( + "warning: DXC not found; non-clang suites will not be able to compile shaders. " + "Pass --dxc-path to silence this warning.", + file=sys.stderr, + ) + dxc_path = None + dxc_dir = str(dxc_path.parent) if dxc_path is not None else "" + + # Golden images: prefer explicit path, then installed copy. An empty + # string indicates "not configured" (lit.cfg.py treats it as such); + # avoid resolving an empty pathlib.Path which would yield the cwd. + if args.golden_images is not None: + golden_dir = str(args.golden_images.resolve()) + else: + candidate = script_dir / "golden-images" + golden_dir = str(candidate.resolve()) if candidate.is_dir() else "" + + for suite in args.suites: + d3d12, vk, mtl, clang, warp, _ = SUITES[suite] + suite_obj_root = output_base + suite_dir = output_base / "test" / suite + suite_dir.mkdir(parents=True, exist_ok=True) + + substitutions = { + "OFFLOADTEST_BINARY_DIR": str(suite_obj_root), + # The directory containing test/lit.cfg.py. In an in-tree build + # this is the project source dir; in the standalone install + # layout it's the install share dir. + "OFFLOADTEST_TEST_ROOT": str(test_src_root.parent), + "LLVM_TOOLS_DIR": str(bin_dir), + "DXC_EXECUTABLE": str(dxc_path) if dxc_path is not None else "", + "SUPPORTS_SPIRV": "True" if vk else "False", + "FORCE_CLANG": "True" if clang else "False", + "FORCE_WARP": "True" if warp else "False", + "WARP_ARCHITECTURE": args.warp_arch, + "DXC_DIR": dxc_dir, + "GOLDENIMAGE_DIR": golden_dir, + "suite": suite, + "TEST_d3d12": "True" if d3d12 else "False", + "TEST_vk": "True" if vk else "False", + "TEST_mtl": "True" if mtl else "False", + "CMAKE_SYSTEM_NAME": args.os_name, + "OFFLOADTEST_ENABLE_DEBUG": "1" if args.enable_debug else "0", + "OFFLOADTEST_ENABLE_VALIDATION": "1" if args.enable_validation else "0", + } + + site_cfg = suite_dir / "lit.site.cfg.py" + site_cfg.write_text( + emit_site_cfg(template_text, substitutions), encoding="utf-8" + ) + print("configured: {}".format(site_cfg)) + + print("\nrun with:") + if len(args.suites) == 1: + print(" lit {}".format(output_base / "test" / args.suites[0])) + else: + print(" lit {}".format(output_base / "test")) + return 0 + + +if __name__ == "__main__": + sys.exit(main())