Skip to content

Unpack LLVM Distribution, and setup lit to run tests for specific test suite #1149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bob80905 merged 21 commits into from
May 20, 2026
Merged

Unpack LLVM Distribution, and setup lit to run tests for specific test suite #1149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
2295213
patch ninja exe path to whatever is in test runner env, and foolproof…
bob80905 May 4, 2026
2951641
another self review
bob80905 May 6, 2026
9fccc9f
address Deric
bob80905 May 8, 2026
36c4d4a
formatting
bob80905 May 8, 2026
eb7ca2d
make separate doc explaining distribution infrastructure, and make se…
bob80905 May 8, 2026
1698781
address Deric, make golden images checkout unconditional
bob80905 May 8, 2026
417d14a
change /U windows backslash, and fix up other golden image problems
bob80905 May 9, 2026
4e8b91b
iterate
bob80905 May 11, 2026
df58adc
iterate
bob80905 May 11, 2026
7f07999
iterate
bob80905 May 11, 2026
16dc95b
iterate
bob80905 May 12, 2026
91ad0d6
remove redundance comment, make it cmakelists specific
bob80905 May 12, 2026
55ae0f1
address Deric
bob80905 May 12, 2026
8746896
separate bin from lib
bob80905 May 12, 2026
13e9f9a
iterate
bob80905 May 13, 2026
0ed120e
iterate, on extraction the dlls and exes should be together to preven…
bob80905 May 13, 2026
853ec69
address Deric, simplify / skip install for DXC
bob80905 May 13, 2026
be3c890
package pdbs too
bob80905 May 13, 2026
0a47a3c
account for Release configs lacking pdbs and erroring the copy step
bob80905 May 13, 2026
930d616
add DEPENDS
bob80905 May 14, 2026
f193a10
address Farzon
bob80905 May 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 141 additions & 62 deletions .github/workflows/build-and-test-callable.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ on:
required: false
default: 'main'
type: string
OffloadTest-fork:
description: 'Test Suite fork'
required: false
default: 'llvm'
type: string
LLVM-branch:
description: 'LLVM Branch'
required: false
Expand Down Expand Up @@ -71,6 +76,11 @@ on:
required: false
default: 'main'
type: string
OffloadTest-fork:
description: 'Test Suite Fork'
required: false
default: 'llvm'
type: string
LLVM-branch:
description: 'LLVM Branch'
required: false
Expand Down Expand Up @@ -147,13 +157,12 @@ jobs:
- name: Checkout OffloadTest
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/offload-test-suite
repository: ${{ inputs.OffloadTest-fork }}/offload-test-suite
ref: ${{ inputs.OffloadTest-branch }}
path: OffloadTest
fetch-depth: 1
- name: Checkout Golden Images
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
if: inputs.SplitBuild != true
with:
repository: llvm/offload-golden-images
ref: main
Expand Down Expand Up @@ -190,9 +199,9 @@ jobs:
cd llvm-project
mkdir build
cd build
cmake -G Ninja ${{ inputs.LLVM-ExtraCMakeArgs }} -DCMAKE_BUILD_TYPE=${{ inputs.BuildType }} -DLLVM_ENABLE_ASSERTIONS=On -C ${{ github.workspace }}/llvm-project/clang/cmake/caches/HLSL.cmake -C ${{ github.workspace }}/OffloadTest/cmake/caches/sccache.cmake -DDXC_DIR=${{ github.workspace }}/DXC/build/bin -DLLVM_EXTERNAL_OFFLOADTEST_SOURCE_DIR=${{ github.workspace }}/OffloadTest -DLLVM_EXTERNAL_PROJECTS="OffloadTest" -DLLVM_LIT_ARGS="--xunit-xml-output=testresults.xunit.xml -v" -DOFFLOADTEST_TEST_CLANG=${{steps.Test-Clang.outputs.TEST_CLANG || 'Off' }} -DGOLDENIMAGE_DIR=${{ github.workspace }}/golden-images ${{ github.workspace }}/llvm-project/llvm/
cmake -G Ninja ${{ inputs.LLVM-ExtraCMakeArgs }} -DCMAKE_BUILD_TYPE=${{ inputs.BuildType }} -DLLVM_ENABLE_ASSERTIONS=On -DLLVM_EXTERNAL_PROJECTS="OffloadTest" -DHLSL_ENABLE_OFFLOAD_DISTRIBUTION=${{ inputs.SplitBuild == true && 'On' || 'Off' }} -C ${{ github.workspace }}/llvm-project/clang/cmake/caches/HLSL.cmake -C ${{ github.workspace }}/OffloadTest/cmake/caches/sccache.cmake -DDXC_DIR=${{ github.workspace }}/DXC/build/bin -DLLVM_EXTERNAL_OFFLOADTEST_SOURCE_DIR=${{ github.workspace }}/OffloadTest -DLLVM_LIT_ARGS="--xunit-xml-output=testresults.xunit.xml -v" -DOFFLOADTEST_TEST_CLANG=${{steps.Test-Clang.outputs.TEST_CLANG || 'Off' }} -DGOLDENIMAGE_DIR=${{ github.workspace }}/golden-images -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ github.workspace }}/llvm-project/llvm/
ninja hlsl-test-depends
- name: Dump GPU Info
- name: Dump GPU Info (build runner)
Copy link
Copy Markdown
Contributor Author

@bob80905 bob80905 May 8, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note, though it doesn't seem to make sense long term to have a dump gpu info step on the build runner machine, it does make sense to have it short term, since PRs will continue to run with splitbuild disabled while I test and validate that everything is working correctly.

if: inputs.SplitBuild != true
run: |
cd llvm-project
Expand Down Expand Up @@ -232,26 +241,83 @@ jobs:
name: dxdiag-${{ inputs.SKU }}-${{inputs.TestTarget}}.txt
path: ${{ runner.temp }}/dxdiag.txt

# When SplitBuild is true, package and upload build
# artifacts for the test job.
- name: Package build artifacts
# When SplitBuild is true, install the LLVM/DXC distribution and the
# standalone offload test suite into a single portable prefix, then
# tar that prefix for the test job. The prefix is fully self-contained:
# tools under bin/, test sources + configure script under
# share/hlsl-test-suite/. No CMake, ninja, or compiler toolchain is
# required on the test runner.
- name: Install distribution
if: inputs.SplitBuild == true
shell: bash
run: |
set -euxo pipefail
cd $GITHUB_WORKSPACE/llvm-project/build
ninja install-distribution install-offload-tools install-offload-test-suite
# Stage DXC into a portable bin/+lib/ tree at dxc-dist.
# See docs/offload-distribution.md ("DXC prefix") for the
# rationale (DXC's install targets don't cover everything we
# need, so we cherry-pick from the build directory).
buildbin=$GITHUB_WORKSPACE/DXC/build/bin
buildlib=$GITHUB_WORKSPACE/DXC/build/lib
distbin=$GITHUB_WORKSPACE/dxc-dist/bin
distlib=$GITHUB_WORKSPACE/dxc-dist/lib
mkdir -p "$distbin" "$distlib"
case "$RUNNER_OS" in
Windows)
cp "$buildbin/dxc.exe" "$distbin/"
cp "$buildbin/dxv.exe" "$distbin/"
cp "$buildbin/dxcompiler.dll" "$distbin/"
cp "$buildbin/dxil.dll" "$distbin/"
Comment on lines +268 to +271
Copy link
Copy Markdown
Member

@farzonl farzonl May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm curious why we copy here instead of using cmake --install I do think we have a special install-dxcompiler flag.

Copy link
Copy Markdown
Contributor Author

@bob80905 bob80905 May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cmake --install / ninja install doesn't work here: it walks every cmake_install.cmake in the build
tree, including LLVM tools (e.g. llvm-as) that aren't built by our default ninja invocation, and aborts on the first
missing artifact.
Deric pointed this out too, and I documented why this is done instead in the docs file.

# PDBs only exist for configs that emit /Zi (Debug,
# RelWithDebInfo). Release / MinSizeRel produce no
# .pdb files, so we skip the copy in those cases.
case "${{ inputs.BuildType }}" in
Debug|RelWithDebInfo)
cp "$buildbin/dxc.pdb" "$distbin/"
cp "$buildbin/dxv.pdb" "$distbin/"
cp "$buildbin/dxcompiler.pdb" "$distbin/"
cp "$buildbin/dxil.pdb" "$distbin/" ;;
esac
cp "$buildlib/dxcompiler.lib" "$distlib/"
cp "$buildlib/dxil.lib" "$distlib/" ;;
Comment thread
Icohedron marked this conversation as resolved.
Linux)
cp "$buildbin/dxc" "$distbin/"
cp "$buildbin/dxv" "$distbin/"
cp "$buildlib/libdxcompiler.so" "$distlib/"
cp "$buildlib/libdxil.so" "$distlib/" ;;
macOS)
cp "$buildbin/dxc" "$distbin/"
cp "$buildbin/dxv" "$distbin/"
cp "$buildlib/libdxcompiler.dylib" "$distlib/"
cp "$buildlib/libdxil.dylib" "$distlib/" ;;
*)
echo "Unsupported RUNNER_OS=$RUNNER_OS" >&2; exit 1 ;;
esac
- name: Package install prefix
if: inputs.SplitBuild == true
shell: bash
run: |
cd $GITHUB_WORKSPACE
tar cf $RUNNER_TEMP/build-artifacts.tar \
--exclude='*.obj' \
--exclude='*.o' \
--exclude='*.ilk' \
--exclude='*.pdb' \
DXC/build/bin llvm-project/build
# Use stdout redirection so tar doesn't try to parse the
# destination Windows drive letter (e.g. C:\...) as a remote
# host:path tuple.
tar cf - -C install . > "$RUNNER_TEMP/build-artifacts.tar"
tar cf - -C dxc-dist . > "$RUNNER_TEMP/dxc-artifacts.tar"
- name: Upload build artifacts
if: inputs.SplitBuild == true
uses: actions/upload-artifact@v4
with:
name: build-${{ inputs.SKU }}-${{ inputs.TestTarget }}
path: ${{ runner.temp }}/build-artifacts.tar
retention-days: 1
- name: Upload DXC artifacts
if: inputs.SplitBuild == true
uses: actions/upload-artifact@v4
with:
name: dxc-${{ inputs.SKU }}-${{ inputs.TestTarget }}
path: ${{ runner.temp }}/dxc-artifacts.tar
retention-days: 1

test:
if: inputs.SplitBuild == true
Expand All @@ -260,53 +326,30 @@ jobs:
checks: write
runs-on: [self-hosted, "hlsl-${{ inputs.SKU }}"]
steps:
- name: Checkout DXC
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: Microsoft/DirectXShaderCompiler
ref: ${{ inputs.DXC-branch }}
path: DXC
fetch-depth: 1
submodules: true
- name: Checkout LLVM
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: ${{ inputs.LLVM-fork }}/llvm-project
ref: ${{ inputs.LLVM-branch }}
path: llvm-project
fetch-depth: 1
- name: Checkout OffloadTest
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/offload-test-suite
ref: ${{ inputs.OffloadTest-branch }}
path: OffloadTest
fetch-depth: 1
- name: Checkout Golden Images
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
with:
repository: llvm/offload-golden-images
ref: main
path: golden-images
fetch-depth: 1
# Test job consumes ONLY the install tarball produced by the build job.
# No source checkouts of llvm-project / DXC / OffloadTest are needed
# here -- everything required to run lit lives inside the prefix.
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: build-${{ inputs.SKU }}-${{ inputs.TestTarget }}
path: ${{ runner.temp }}
- name: Extract build artifacts
- name: Download DXC artifacts
uses: actions/download-artifact@v4
with:
name: dxc-${{ inputs.SKU }}-${{ inputs.TestTarget }}
path: ${{ runner.temp }}
- name: Extract install prefix
shell: bash
run: |
set -euxo pipefail
cd $GITHUB_WORKSPACE
tar xf $RUNNER_TEMP/build-artifacts.tar
- name: Refresh artifact timestamps
shell: python
run: |
import pathlib
for d in ['DXC/build/bin', 'llvm-project/build']:
for f in pathlib.Path(d).rglob('*'):
if f.is_file():
f.touch()
mkdir -p install dxc-dist
# Pipe the archive in via stdin and use a relative -C path so
# tar doesn't try to parse the Windows drive letter or treat
# backslashes as escape characters.
tar xf - -C install < "$RUNNER_TEMP/build-artifacts.tar"
tar xf - -C dxc-dist < "$RUNNER_TEMP/dxc-artifacts.tar"
- name: Setup Windows x64
if: inputs.OS == 'windows' && runner.arch != 'ARM64'
uses: llvm/actions/setup-windows@89a8cf80982d830faab019237860b344a6390c30 # main
Expand All @@ -320,24 +363,60 @@ jobs:
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
pip-install: -r ${{ github.workspace }}/OffloadTest/test/requirements.txt
- name: Dump GPU Info
# The test runner needs only lit + pyyaml; no source checkout is
# available so we can't pip install -r requirements.txt. Keep
# this list in sync with OffloadTest/test/requirements.txt.
pip-install: lit pyyaml
- name: Dump GPU Info (test runner)
shell: bash
run: |
cd llvm-project
cd build
./bin/api-query
$GITHUB_WORKSPACE/install/bin/api-query
- name: Resolve lit suite from TestTarget
id: suite
shell: bash
run: |
# SplitBuild callers always pass a single specific suite as
# TestTarget=check-hlsl-<suite>. Strip the prefix to get the
# suite name configure-test-suite.py understands.
set -euxo pipefail
target='${{ inputs.TestTarget }}'
case "$target" in
check-hlsl-*)
suite="${target#check-hlsl-}"
;;
*)
echo "::error::SplitBuild requires TestTarget=check-hlsl-<suite>, got: $target"
exit 1
;;
esac
echo "suite=$suite" >> $GITHUB_OUTPUT
echo "Will run lit suite: $suite"
- name: Configure standalone test suite
shell: bash
run: |
set -euxo pipefail
cd "$GITHUB_WORKSPACE/install/share/hlsl-test-suite"
# Locate dxc in the separate DXC install prefix. .exe on Windows,
# bare name elsewhere.
dxc_path="$GITHUB_WORKSPACE/dxc-dist/bin/dxc"
if [ -x "${dxc_path}.exe" ]; then dxc_path="${dxc_path}.exe"; fi
Copy link
Copy Markdown
Member

@farzonl farzonl May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm is dxc_path sanitized. I've had issues with paths with bash on windows. Concatenation is highly vulnerable to escaping errors on Windows and we are concating GITHUB_WORKSPACE which makes me worried.

python configure-test-suite.py \
--suite "${{ steps.suite.outputs.suite }}" \
--dxc-path "$dxc_path"
- name: Run HLSL Tests
shell: bash
run: |
cd llvm-project
cd build
ninja check-hlsl-unit
ninja ${{ inputs.TestTarget }}
set -euxo pipefail
cd $GITHUB_WORKSPACE/install/share/hlsl-test-suite
suite='${{ steps.suite.outputs.suite }}'
lit -v "run/test/$suite" \
--xunit-xml-output="run/test/$suite/testresults.xunit.xml"
- name: Publish Test Results
uses: EnricoMi/publish-unit-test-result-action/macos@34d7c956a59aed1bfebf31df77b8de55db9bbaaf # v2.21.0
if: always() && inputs.OS == 'macOS'
with:
comment_mode: off
files: llvm-project/build/**/testresults.xunit.xml
files: ${{ github.workspace }}/install/share/hlsl-test-suite/run/test/**/testresults.xunit.xml
- name: Run dxdiag (Windows only)
if: inputs.OS == 'windows' && failure()
shell: powershell
Expand Down
17 changes: 17 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include
macro(add_offloadtest_tool name)
add_llvm_executable(${name} ${ARGN})
set_target_properties(${name} PROPERTIES FOLDER "HLSL Test/Tools")
# Install offload-test tools under a single umbrella component so the
# standalone test distribution can pull them all with one
# `install-offload-tools` target. LLVM_TOOLS_INSTALL_DIR defaults to
# `bin` when building inside an LLVM tree.
install(TARGETS ${name}
COMPONENT offload-tools
RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}")
endmacro()

macro(add_offloadtest_library name)
Expand Down Expand Up @@ -132,3 +139,13 @@ add_subdirectory(tools)

add_subdirectory(unittests)
add_subdirectory(test)

# Aggregate install target for the offload-tools component. After the
# `add_subdirectory(tools)` call above has registered every tool under the
# `offload-tools` component via `add_offloadtest_tool`, this provides a
# single `install-offload-tools` target that builds + installs all of them.
if (NOT LLVM_ENABLE_IDE)
add_llvm_install_targets(install-offload-tools
COMPONENT offload-tools
DEPENDS api-query imgdiff offloader)
Copy link
Copy Markdown
Member

@farzonl farzonl May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this the right way to do this in Cmake? don't you use the env names? While unlikely if you use the cmake definition for these executable names instead of the executable name it allows us to change the name of the executable in only one place instead of updated across multiple cmake files.

Copy link
Copy Markdown
Contributor Author

@bob80905 bob80905 May 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the convention is to define tool targets as variables. E.g., opt, llc, llvm-as, etc are referenced literally, so I think this is the right convention.

endif()
Loading
Loading