From 39e91098b58bfbef973bc759803fbfcc6064b42b Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Thu, 16 Mar 2023 11:42:57 -0500 Subject: [PATCH] [libc] Enable integration tests targeting the GPU This patch enables integration tests running on the GPU. This uses the RPC interface implemented in D145913 to compile the necessary dependencies for the integration test object. We can then use this to compile the objects for the GPU directly and execute them using the AMD HSA loader combined with its RPC server. For example, the compiler is performing the following actions to execute the integration tests. ``` $ clang++ --target=amdgcn-amd-amdhsa -mcpu=gfx1030 -nostdlib -flto -ffreestanding \ crt1.o io.o quick_exit.o test.o rpc_client.o args_test.o -o image $ ./amdhsa_loader image 1 2 5 args_test.cpp:24: Expected 'my_streq(argv[3], "3")' to be true, but is false ``` This currently only works with a single threaded client implementation running on AMDGPU. Further work will implement multiple clients for AMD and the ability to run on NVPTX as well. Depends on D145913 Reviewed By: sivachandra, JonChesterfield Differential Revision: https://reviews.llvm.org/D146256 --- libc/CMakeLists.txt | 2 +- libc/cmake/modules/LLVMLibCTestRules.cmake | 29 +++++++++++++--- libc/startup/gpu/amdgpu/CMakeLists.txt | 9 +++-- libc/test/CMakeLists.txt | 34 +++++++++++-------- libc/test/IntegrationTest/CMakeLists.txt | 16 +++++++++ .../integration/startup/gpu/CMakeLists.txt | 11 ++++++ .../integration/startup/gpu/args_test.cpp | 27 +++++++++++++++ libc/utils/gpu/loader/CMakeLists.txt | 13 +++++++ 8 files changed, 120 insertions(+), 21 deletions(-) create mode 100644 libc/test/integration/startup/gpu/CMakeLists.txt create mode 100644 libc/test/integration/startup/gpu/args_test.cpp diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt index 9fd9bce66bd5..0a9627246d0f 100644 --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -197,7 +197,7 @@ endif() # of the other directories. # TODO: Add testing support for the libc GPU target. add_subdirectory(lib) -if(LLVM_INCLUDE_TESTS AND NOT LIBC_TARGET_ARCHITECTURE_IS_GPU) +if(LLVM_INCLUDE_TESTS) add_subdirectory(test) add_subdirectory(fuzzing) endif() diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake index bf862b743c8a..f28b345fd45c 100644 --- a/libc/cmake/modules/LLVMLibCTestRules.cmake +++ b/libc/cmake/modules/LLVMLibCTestRules.cmake @@ -412,7 +412,8 @@ endfunction(add_libc_fuzzer) # targets added with add_entrypoint_object or add_object_library. function(add_integration_test test_name) get_fq_target_name(${test_name} fq_target_name) - if(NOT (${LIBC_TARGET_OS} STREQUAL "linux")) + set(supported_targets gpu linux) + if(NOT (${LIBC_TARGET_OS} IN_LIST supported_targets)) message(STATUS "Skipping ${fq_target_name} as it is not available on ${LIBC_TARGET_OS}.") return() endif() @@ -438,9 +439,10 @@ function(add_integration_test test_name) get_fq_deps_list(fq_deps_list ${INTEGRATION_TEST_DEPENDS}) list(APPEND fq_deps_list - # All integration tests use the operating system's startup object and need - # to inherit the same dependencies. + # All integration tests use the operating system's startup object with the + # integration test object and need to inherit the same dependencies. libc.startup.${LIBC_TARGET_OS}.crt1 + libc.test.IntegrationTest.test # We always add the memory functions objects. This is because the # compiler's codegen can emit calls to the C memory functions. libc.src.string.bcmp @@ -496,16 +498,35 @@ function(add_integration_test test_name) ) target_compile_options(${fq_build_target_name} PRIVATE -fpie -ffreestanding ${INTEGRATION_TEST_COMPILE_OPTIONS}) + # The GPU build requires overriding the default CMake triple and architecture. + if(LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU) + target_compile_options(${fq_build_target_name} PRIVATE + -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} -emit-llvm + --target=${LIBC_GPU_TARGET_TRIPLE}) + elseif(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX) + target_compile_options(${fq_build_target_name} PRIVATE + -march=${LIBC_GPU_TARGET_ARCHITECTURE} + --target=${LIBC_GPU_TARGET_TRIPLE}) + endif() + target_link_options(${fq_build_target_name} PRIVATE -nostdlib -static) target_link_libraries(${fq_build_target_name} ${fq_target_name}.__libc__ + libc.startup.${LIBC_TARGET_OS}.crt1 libc.test.IntegrationTest.test) add_dependencies(${fq_build_target_name} libc.test.IntegrationTest.test ${INTEGRATION_TEST_DEPENDS}) + # Tests on the GPU require an external loader utility to launch the kernel. + if(TARGET libc.utils.gpu.loader) + get_target_property(gpu_loader_exe libc.utils.gpu.loader "EXECUTABLE") + endif() + add_custom_target( ${fq_target_name} - COMMAND ${INTEGRATION_TEST_ENV} $ ${INTEGRATION_TEST_ARGS} + COMMAND ${INTEGRATION_TEST_ENV} + $<$:${gpu_loader_exe}> + $ ${INTEGRATION_TEST_ARGS} COMMENT "Running integration test ${fq_target_name}" ) add_dependencies(${INTEGRATION_TEST_SUITE} ${fq_target_name}) diff --git a/libc/startup/gpu/amdgpu/CMakeLists.txt b/libc/startup/gpu/amdgpu/CMakeLists.txt index d1c6fc7cd644..891d20993b08 100644 --- a/libc/startup/gpu/amdgpu/CMakeLists.txt +++ b/libc/startup/gpu/amdgpu/CMakeLists.txt @@ -16,5 +16,10 @@ add_startup_object( get_fq_target_name(crt1 fq_name) # Ensure that clang uses the correct linker for this object type. -target_link_libraries(${fq_name} PUBLIC - "--target=${LIBC_GPU_TARGET_TRIPLE}" "-flto") +target_link_libraries( + ${fq_name} + PUBLIC + "-mcpu=${LIBC_GPU_TARGET_ARCHITECTURE}" + "--target=${LIBC_GPU_TARGET_TRIPLE}" + "-flto" +) diff --git a/libc/test/CMakeLists.txt b/libc/test/CMakeLists.txt index d325aac68f52..8a7023c3fff6 100644 --- a/libc/test/CMakeLists.txt +++ b/libc/test/CMakeLists.txt @@ -1,9 +1,9 @@ -add_subdirectory(UnitTest) +add_custom_target(check-libc) +add_custom_target(libc-unit-tests) +add_dependencies(check-libc libc-unit-tests) -if(LLVM_LIBC_FULL_BUILD AND NOT - (LIBC_TARGET_ARCHITECTURE_IS_GPU OR LIBC_TARGET_OS_IS_BAREMETAL)) - add_subdirectory(IntegrationTest) -endif() +add_custom_target(exhaustive-check-libc) +add_custom_target(libc-long-running-tests) add_header_library( errno_setter_matcher @@ -13,22 +13,28 @@ add_header_library( libc.src.errno.errno ) -add_custom_target(check-libc) -add_custom_target(libc-unit-tests) -add_dependencies(check-libc libc-unit-tests) +if(NOT TARGET libc.utils.gpu.loader OR NOT TARGET libc.startup.gpu.crt1) + message(WARNING "Cannot build libc GPU tests, missing loader implementation") + return() +endif() -add_custom_target(exhaustive-check-libc) -add_custom_target(libc-long-running-tests) +if(NOT LIBC_TARGET_ARCHITECTURE_IS_GPU) + add_subdirectory(UnitTest) + add_subdirectory(src) + add_subdirectory(utils) +endif() -add_subdirectory(src) -add_subdirectory(utils) +if(LLVM_LIBC_FULL_BUILD AND NOT LIBC_TARGET_OS_IS_BAREMETAL) + add_subdirectory(IntegrationTest) +endif() if(NOT LLVM_LIBC_FULL_BUILD) return() endif() -if(NOT ${LIBC_TARGET_OS} STREQUAL "linux") - # Integration tests are currently only available for linux. +if(NOT ${LIBC_TARGET_OS} STREQUAL "linux" AND + NOT ${LIBC_TARGET_OS} STREQUAL "gpu") + # Integration tests are currently only available for linux and the GPU. return() endif() add_subdirectory(integration) diff --git a/libc/test/IntegrationTest/CMakeLists.txt b/libc/test/IntegrationTest/CMakeLists.txt index 3d8a3497bbb2..62bd114645b5 100644 --- a/libc/test/IntegrationTest/CMakeLists.txt +++ b/libc/test/IntegrationTest/CMakeLists.txt @@ -1,9 +1,25 @@ +if(LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU) + set(TEST_COMPILE_FLAGS + -mcpu=${LIBC_GPU_TARGET_ARCHITECTURE} + -emit-llvm # AMDGPU's intermediate object file format is bitcode. + --target=${LIBC_GPU_TARGET_TRIPLE} + ) +elseif(LIBC_GPU_TARGET_ARCHITECTURE_IS_NVPTX) + set(TEST_COMPILE_FLAGS + -march=${LIBC_GPU_TARGET_ARCHITECTURE} + --target=${LIBC_GPU_TARGET_TRIPLE} + ) +endif() + add_object_library( test SRCS test.cpp + COMPILE_OPTIONS + ${TEST_COMPILE_FLAGS} HDRS test.h DEPENDS libc.src.__support.OSUtil.osutil + NO_GPU_BUNDLE # Compile this file directly without special GPU handling. ) diff --git a/libc/test/integration/startup/gpu/CMakeLists.txt b/libc/test/integration/startup/gpu/CMakeLists.txt new file mode 100644 index 000000000000..5451a27c2887 --- /dev/null +++ b/libc/test/integration/startup/gpu/CMakeLists.txt @@ -0,0 +1,11 @@ +add_custom_target(libc-startup-tests) +add_dependencies(libc-integration-tests libc-startup-tests) + +add_integration_test( + startup_args_test + SUITE libc-startup-tests + SRCS + args_test.cpp + ARGS + 1 2 3 +) diff --git a/libc/test/integration/startup/gpu/args_test.cpp b/libc/test/integration/startup/gpu/args_test.cpp new file mode 100644 index 000000000000..f3a5410691c2 --- /dev/null +++ b/libc/test/integration/startup/gpu/args_test.cpp @@ -0,0 +1,27 @@ +//===-- Loader test to check args to main ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/IntegrationTest/test.h" + +static bool my_streq(const char *lhs, const char *rhs) { + const char *l, *r; + for (l = lhs, r = rhs; *l != '\0' && *r != '\0'; ++l, ++r) + if (*l != *r) + return false; + + return *l == '\0' && *r == '\0'; +} + +TEST_MAIN(int argc, char **argv) { + ASSERT_TRUE(argc == 4); + ASSERT_TRUE(my_streq(argv[1], "1")); + ASSERT_TRUE(my_streq(argv[2], "2")); + ASSERT_TRUE(my_streq(argv[3], "3")); + + return 0; +} diff --git a/libc/utils/gpu/loader/CMakeLists.txt b/libc/utils/gpu/loader/CMakeLists.txt index f66a9a2e2aae..f643bfcd9abb 100644 --- a/libc/utils/gpu/loader/CMakeLists.txt +++ b/libc/utils/gpu/loader/CMakeLists.txt @@ -4,4 +4,17 @@ target_include_directories(gpu_loader PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm) if(hsa-runtime64_FOUND) add_subdirectory(amdgpu) +else() + message(STATUS "Skipping HSA loader for gpu target, no HSA was detected") +endif() + +# Add a custom target to be used for testing. +if(TARGET amdhsa_loader AND LIBC_GPU_TARGET_ARCHITECTURE_IS_AMDGPU) + add_custom_target(libc.utils.gpu.loader) + add_dependencies(libc.utils.gpu.loader amdhsa_loader) + set_target_properties( + libc.utils.gpu.loader + PROPERTIES + EXECUTABLE "$" + ) endif()