From 09a1df614e2540246e240ca881662a44db4779ae Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Tue, 14 Jul 2026 07:53:20 -0400 Subject: [PATCH 1/4] Add builds for remaining generator stub tests --- test/generator/CMakeLists.txt | 136 ++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 7 deletions(-) diff --git a/test/generator/CMakeLists.txt b/test/generator/CMakeLists.txt index 3b681a87d9d8..92dd9bc52ac3 100644 --- a/test/generator/CMakeLists.txt +++ b/test/generator/CMakeLists.txt @@ -121,7 +121,9 @@ endfunction() # function(_add_halide_libraries TARGET) set(options GRADIENT_DESCENT OMIT_C_BACKEND) - set(oneValueArgs FROM GENERATOR_NAME FUNCTION_NAME NAMESPACE USE_RUNTIME AUTOSCHEDULER) + set(oneValueArgs + FROM GENERATOR_NAME FUNCTION_NAME NAMESPACE USE_RUNTIME AUTOSCHEDULER REGISTRATION + ) set(multiValueArgs ENABLE_IF TARGETS FEATURES PARAMS PLUGINS EXTERNS) cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -136,6 +138,15 @@ function(_add_halide_libraries TARGET) set(GRADIENT_DESCENT_OPT "") endif () + # REGISTRATION is one of add_halide_library's "extra output" args, which warns if passed + # with an empty value (as opposed to NAMESPACE/USE_RUNTIME/AUTOSCHEDULER/etc., which are + # fine empty) -- so only pass it through when actually requested. + if (args_REGISTRATION) + set(REGISTRATION_OPT REGISTRATION "${args_REGISTRATION}") + else () + set(REGISTRATION_OPT "") + endif () + # Fill in default values for some arguments, as needed if (NOT args_FROM) add_halide_generator("${TARGET}.generator" SOURCES "${TARGET}_generator.cpp") @@ -163,7 +174,11 @@ function(_add_halide_libraries TARGET) PARAMS "${args_PARAMS}" PLUGINS "${args_PLUGINS}" FUNCTION_INFO_HEADER function_info_header_out + ${REGISTRATION_OPT} ) + if (args_REGISTRATION) + set(${args_REGISTRATION} "${${args_REGISTRATION}}" PARENT_SCOPE) + endif () if (args_EXTERNS) target_link_libraries(${TARGET} INTERFACE ${args_EXTERNS}) endif () @@ -355,11 +370,13 @@ endfunction() _add_halide_libraries(acquire_release) _add_halide_aot_tests(acquire_release LINK_TO_GPU) -# TODO: what are these? # configure_jittest.cpp # example_jittest.cpp # registration_test.cpp # rungen_test.cpp +# These depend on generator/library targets (blur2x2, cxx_mangling, pyramid, example, +# configure) that aren't defined until later in this file; see the bottom of this file, +# just after the stubuser block, for where they're actually wired up. # alias_aottest.cpp # alias_generator.cpp @@ -433,7 +450,7 @@ _add_halide_aot_tests(bit_operations) # blur2x2_aottest.cpp # blur2x2_generator.cpp -_add_halide_libraries(blur2x2) +_add_halide_libraries(blur2x2 REGISTRATION blur2x2_registration_cpp) _add_halide_aot_tests(blur2x2) # buffer_copy_aottest.cpp @@ -460,6 +477,7 @@ _add_halide_aot_tests( # configure_generator.cpp _add_halide_libraries(configure) _add_halide_aot_tests(configure) +add_halide_stub(configure.stub FROM configure.generator GENERATOR configure) # TODO: this could be made to work under wasm with some tweaking, it's just build-rule complexity if (NOT ${_USING_WASM}) @@ -473,6 +491,7 @@ if (NOT ${_USING_WASM}) FUNCTION_NAME HalideTest::AnotherNamespace::cxx_mangling FEATURES c_plus_plus_name_mangling EXTERNS cxx_mangling_externs + REGISTRATION cxx_mangling_registration_cpp ) _add_halide_aot_tests(cxx_mangling) if ("NVPTX" IN_LIST Halide_LLVM_COMPONENTS AND Halide_TARGET MATCHES "cuda") @@ -524,7 +543,7 @@ _add_halide_aot_tests(error_codes) # example_aottest.cpp # example_generator.cpp -_add_halide_libraries(example) +_add_halide_libraries(example REGISTRATION example_registration_cpp) _add_halide_aot_tests(example GROUPS multithreaded) # extern_output_aottest.cpp @@ -748,7 +767,7 @@ _add_halide_aot_tests(output_assign) # pyramid_aottest.cpp # pyramid_generator.cpp -_add_halide_libraries(pyramid PARAMS levels=10) +_add_halide_libraries(pyramid PARAMS levels=10 REGISTRATION pyramid_registration_cpp) _add_halide_aot_tests(pyramid GROUPS multithreaded) # rdom_input_aottest.cpp @@ -764,11 +783,114 @@ _add_halide_aot_tests(string_param) # stubtest_aottest.cpp # stubtest_generator.cpp # stubtest_jittest.cpp -# TODO: stubs not supported in CMake +add_halide_generator(stubtest.generator SOURCES stubtest_generator.cpp) +add_halide_stub(stubtest.stub FROM stubtest.generator GENERATOR stubtest) + +add_executable(generator_jit_stubtest stubtest_jittest.cpp stubtest_generator.cpp) +target_link_libraries(generator_jit_stubtest PRIVATE Halide::Test stubtest.stub) +add_halide_test(generator_jit_stubtest GROUPS generator) + +_add_halide_libraries( + stubtest + FROM stubtest.generator + PARAMS + untyped_buffer_input.type=uint8 + untyped_buffer_input.dim=3 + simple_input.type=float32 + array_input.type=float32 + array_input.size=2 + int_arg.size=2 + tuple_output.type=float32,float32 + vectorize=true + OMIT_C_BACKEND +) +_add_halide_aot_tests(stubtest OMIT_C_BACKEND) # stubuser_aottest.cpp # stubuser_generator.cpp -# TODO: stubs not supported in CMake +add_halide_generator( + stubuser.generator + SOURCES + stubuser_generator.cpp + stubtest_generator.cpp + configure_generator.cpp + LINK_LIBRARIES configure.stub stubtest.stub +) +_add_halide_libraries(stubuser FROM stubuser.generator OMIT_C_BACKEND) +_add_halide_libraries( + stubuser_auto + ENABLE_IF WITH_AUTOSCHEDULERS + FROM stubuser.generator + GENERATOR_NAME stubuser + FUNCTION_NAME stubuser_auto + AUTOSCHEDULER Halide::Mullapudi2016 + PLUGINS Halide::Mullapudi2016 + OMIT_C_BACKEND +) +_add_halide_aot_tests( + stubuser + ENABLE_IF WITH_AUTOSCHEDULERS AND NOT ${_USING_WASM} + HALIDE_LIBRARIES stubuser stubuser_auto + OMIT_C_BACKEND +) + +# configure_jittest.cpp +# (configure.stub was already created FROM configure.generator, above, for stubuser) +add_executable(generator_jit_configure configure_jittest.cpp configure_generator.cpp) +target_link_libraries(generator_jit_configure PRIVATE Halide::Test configure.stub) +add_halide_test(generator_jit_configure GROUPS generator) + +# example_jittest.cpp +add_halide_stub(example.stub FROM example.generator GENERATOR example) +add_executable(generator_jit_example example_jittest.cpp example_generator.cpp) +target_link_libraries(generator_jit_example PRIVATE Halide::Test example.stub) +add_halide_test(generator_jit_example GROUPS generator) + +if (NOT ${_USING_WASM}) + # rungen_test.cpp + # + # Exercises Halide::RunGen::RunGen directly (rather than via the RunGenMain driver), using + # the plain (non-extra-key-value) registration.cpp for "example" to verify that the argv/ + # metadata registration hook fires with the expected function pointers. + add_executable(rungen_test rungen_test.cpp "${example_registration_cpp}") + target_link_libraries(rungen_test PRIVATE example example.runtime Halide::ImageIO Halide::Tools) + add_halide_test(rungen_test GROUPS generator) + + # registration_test.cpp + # + # Tests the registration mechanism (independent of RunGen) by linking three filters' + # registration.cpp files, each compiled to call a per-filter + # halide_register_extra_key_value_pairs_() hook (defined by the test itself), and + # checking that halide_register_argv_and_metadata() is invoked once per filter with the + # expected argv/metadata/key-value-pairs. + add_executable( + registration_test + registration_test.cpp + "${blur2x2_registration_cpp}" + "${cxx_mangling_registration_cpp}" + "${pyramid_registration_cpp}" + ) + set_source_files_properties( + "${blur2x2_registration_cpp}" + PROPERTIES + COMPILE_DEFINITIONS + HALIDE_REGISTER_EXTRA_KEY_VALUE_PAIRS_FUNC=halide_register_extra_key_value_pairs_blur2x2 + ) + set_source_files_properties( + "${cxx_mangling_registration_cpp}" + PROPERTIES + COMPILE_DEFINITIONS + HALIDE_REGISTER_EXTRA_KEY_VALUE_PAIRS_FUNC=halide_register_extra_key_value_pairs_cxx_mangling + ) + set_source_files_properties( + "${pyramid_registration_cpp}" + PROPERTIES + COMPILE_DEFINITIONS + HALIDE_REGISTER_EXTRA_KEY_VALUE_PAIRS_FUNC=halide_register_extra_key_value_pairs_pyramid + ) + target_link_libraries(registration_test PRIVATE blur2x2 cxx_mangling pyramid blur2x2.runtime) + add_halide_test(registration_test GROUPS generator) +endif () # shuffler_aottest.cpp # shuffler_generator.cpp From a5692dc672dc190e8db7329bb37b2fd74d984c90 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Mon, 13 Jul 2026 02:29:40 -0400 Subject: [PATCH 2/4] Add IRMatch throughput performance experiment --- test/performance/CMakeLists.txt | 1 + test/performance/irmatch_throughput.cpp | 106 ++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 test/performance/irmatch_throughput.cpp diff --git a/test/performance/CMakeLists.txt b/test/performance/CMakeLists.txt index cee54a47c132..30492c2a8514 100644 --- a/test/performance/CMakeLists.txt +++ b/test/performance/CMakeLists.txt @@ -20,6 +20,7 @@ tests( fast_sine_cosine.cpp gpu_half_throughput.cpp interleave.cpp + irmatch_throughput.cpp jit_stress.cpp lots_of_inputs.cpp memcpy.cpp diff --git a/test/performance/irmatch_throughput.cpp b/test/performance/irmatch_throughput.cpp new file mode 100644 index 000000000000..b366defb6937 --- /dev/null +++ b/test/performance/irmatch_throughput.cpp @@ -0,0 +1,106 @@ +#include "Halide.h" +#include "halide_benchmark.h" + +#include +#include + +// A matcher-bound throughput benchmark for the term-rewriting engine in +// src/IRMatch.h. Most rules deliberately fail to match, which is the realistic +// hot path (the simplifier tries many rules per node and few fire). + +using namespace Halide; +using namespace Halide::Internal; +using namespace Halide::Tools; + +namespace { + +// Run a large battery of Sub-canonicalization rules, mirroring the structure +// of src/Simplify_Sub.cpp. Returns the number of rules that fired (kept live +// so nothing is optimized away). +HALIDE_ALWAYS_INLINE int try_sub_rules(const Expr &e) { + // Putting this at the top level trips a GCC ADL bug! + using namespace Halide::Internal::IRMatcher; + + auto rewrite = IRMatcher::rewriter(e, e.type()); + + Wild<0> x; + Wild<1> y; + Wild<2> z; + WildConst<0> c0; + WildConst<1> c1; + + return rewrite(x - x, 0) || + rewrite(x - 0, x) || + rewrite(c0 - c1, fold(c0 - c1)) || + rewrite((x + y) - x, y) || + rewrite((x + y) - y, x) || + rewrite((x - y) - x, -y) || + rewrite((x + c0) - c1, x + fold(c0 - c1)) || + rewrite((c0 - x) - c1, fold(c0 - c1) - x) || + rewrite(x - (x + y), -y) || + rewrite(x - (y + x), -y) || + rewrite((x + y) - (x + z), y - z) || + rewrite((x + y) - (z + x), y - z) || + rewrite((y + x) - (x + z), y - z) || + rewrite((y + x) - (z + x), y - z) || + rewrite(min(x, y) - x, min(y - x, 0)) || + rewrite(min(x, y) - y, min(x - y, 0)) || + rewrite(max(x, y) - x, max(y - x, 0)) || + rewrite(max(x, y) - y, max(x - y, 0)) || + rewrite(x - min(x, y), max(x - y, 0)) || + rewrite(x - max(x, y), min(x - y, 0)) || + rewrite((x + c0) - (y + c1), (x - y) + fold(c0 - c1)) || + rewrite((x * c0) - (y * c0), (x - y) * c0) || + rewrite(c0 - (c1 - x), (c0 - c1) + x) || + rewrite(c0 - (x + c1), fold(c0 - c1) - x) || + rewrite((x - y) - (x - z), z - y) || + rewrite((x - y) - (z - y), x - z); +} + +std::vector make_corpus() { + std::vector corpus; + for (Type t : {Int(32), Int(16), UInt(32), Float(32)}) { + Expr x = Variable::make(t, "x"); + Expr y = Variable::make(t, "y"); + Expr z = Variable::make(t, "z"); + Expr c0 = make_const(t, 3); + Expr c1 = make_const(t, 5); + // A mix: some hit late rules, some hit early rules, some fail entirely. + corpus.push_back(x - x); // fires rule 1 + corpus.push_back((x + c0) - (y + c1)); // fires a late rule + corpus.push_back((x + y) - (z + x)); // fires a mid rule + corpus.push_back(max(x, y) - z); // fails everything + corpus.push_back((x * c0) - (y * c0)); // fires a late rule + corpus.push_back(min(x, y) - x); // fires a mid rule + corpus.push_back((x - y) - (z - y)); // fires the last rule + corpus.push_back(Variable::make(t, "w") - z); // fails everything + } + return corpus; +} + +} // namespace + +int main(int argc, char **argv) { + std::vector corpus = make_corpus(); + + // Prevent the optimizer from discarding the work. + volatile int sink = 0; + + // Warmup + sanity. + for (const Expr &e : corpus) { + sink += try_sub_rules(e); + } + + double t = benchmark(20, 200, [&]() { + int fired = 0; + for (const Expr &e : corpus) { + fired += try_sub_rules(e); + } + sink += fired; + }); + + printf("irmatch throughput: %.2f ns per rule-battery (corpus of %d, sink %d)\n", + (t / corpus.size()) * 1e9, (int)corpus.size(), (int)sink); + printf("Success!\n"); + return 0; +} From cd20a615796ec9c09ac6568d08d35e8acbc5364b Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Sun, 12 Jul 2026 12:25:12 -0400 Subject: [PATCH 3/4] Separate the language type from the ABI type - ABI halide_type_t drops `lanes` -> {code, bits, reserved} (still 4 bytes; halide_buffer_t layout unchanged). Halide::Type is decoupled and stores its own fields; must be explicitly erased with `as_abi` (asserts scalar), Trace event/packet carry their own `lanes`. - Halide::Type is reduced to 8 bytes: the handle_type pointer is replaced by a 4-byte index into a process-wide handle-type intern table. - The IR matcher / constant folder uses Halide::Type directly; This keeps handle C++-type metadata fidelity through matcher reconstruction. - CodeGen intrinsic tables (ARM/X86/Hexagon/PowerPC/WebAssembly) use Halide::Type. - Added ostream operator<< support for halide_type_t directly. Some open problems remain: - d3d12 buffer element types treated as scalar (multi-channel texture format selection is a follow-up). - A few instances call for invalid/uninitialized `Type`s, which goes against the overarching goal of making Type always coherent. Fixes #9202 Fixes #9203 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/hannk/interpreter/ops.cpp | 20 +- apps/hannk/util/buffer_util.h | 6 +- apps/hannk/util/error_util.cpp | 14 - apps/hannk/util/error_util.h | 1 - python_bindings/src/halide/halide_/PyBuffer.h | 2 +- .../src/halide/halide_/PyCallable.cpp | 6 +- python_bindings/src/halide/halide_/PyType.cpp | 4 +- src/AddImageChecks.cpp | 2 +- src/Buffer.h | 20 +- src/CPlusPlusMangle.cpp | 50 +-- src/Callable.cpp | 4 +- src/Callable.h | 60 +-- src/CodeGen_ARM.cpp | 10 +- src/CodeGen_C.cpp | 56 +-- src/CodeGen_Hexagon.cpp | 52 +-- src/CodeGen_LLVM.cpp | 10 +- src/CodeGen_PowerPC.cpp | 8 +- src/CodeGen_WebAssembly.cpp | 8 +- src/CodeGen_X86.cpp | 8 +- src/Deinterleave.cpp | 7 +- src/IREquality.cpp | 10 +- src/IRMatch.h | 402 +++++++++--------- src/OffloadGPULoops.cpp | 2 +- src/Param.h | 12 +- src/Pipeline.cpp | 4 +- src/PythonExtensionGen.cpp | 1 - src/Type.cpp | 79 +++- src/Type.h | 133 ++++-- src/WasmExecutor.cpp | 38 +- src/runtime/HalideRuntime.h | 94 ++-- src/runtime/d3d12compute.cpp | 22 +- src/runtime/halide_buffer_t.cpp | 6 +- src/runtime/metal.cpp | 2 +- src/runtime/to_string.cpp | 4 - src/runtime/trace_helper.cpp | 5 +- src/runtime/tracing.cpp | 18 +- src/runtime/webgpu.cpp | 1 - test/correctness/CMakeLists.txt | 1 + test/correctness/compute_with.cpp | 4 +- test/correctness/irmatch.cpp | 67 +++ test/correctness/reschedule.cpp | 2 +- test/correctness/simd_op_check.h | 2 +- test/correctness/simplify.cpp | 12 +- test/correctness/specialize.cpp | 4 +- test/correctness/tracing.cpp | 6 +- test/correctness/tracing_broadcast.cpp | 2 +- test/correctness/tracing_thread_ids.cpp | 4 +- test/correctness/vectorize_guard_with_if.cpp | 2 +- test/generator/metadata_tester_aottest.cpp | 76 ++-- test/generator/stubtest_jittest.cpp | 8 +- tools/RunGen.h | 75 ++-- tools/halide_image_io.h | 124 +++--- tools/halide_trace_config.h | 31 +- util/HalideTraceDump.cpp | 17 +- util/HalideTraceUtils.h | 6 +- util/HalideTraceViz.cpp | 55 +-- 56 files changed, 927 insertions(+), 752 deletions(-) create mode 100644 test/correctness/irmatch.cpp diff --git a/apps/hannk/interpreter/ops.cpp b/apps/hannk/interpreter/ops.cpp index 11a693f58ab4..f7cecd37877e 100644 --- a/apps/hannk/interpreter/ops.cpp +++ b/apps/hannk/interpreter/ops.cpp @@ -584,22 +584,22 @@ double dequantize_scalar(const Tensor *t) { int zero = q.zero.empty() ? 0 : q.zero.front(); const auto &buf = t->buffer(); - switch (buf.type().element_of().as_u32()) { - case halide_type_of().as_u32(): + switch (buf.type()) { + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; - case halide_type_of().as_u32(): + case halide_type_of(): return (as_scalar(buf) - zero) * scale; default: HLOG(FATAL) << "Unsupported type " << buf.type(); @@ -765,7 +765,7 @@ halide_type_t ConvOp::filter_type() const { return metadata->arguments[2].type; } else { HLOG(FATAL) << "Unsupported type " << output()->type() << "\n"; - return halide_type_t(halide_type_int, 0, 0); + return halide_type_t(halide_type_int, 0); } } diff --git a/apps/hannk/util/buffer_util.h b/apps/hannk/util/buffer_util.h index d9559b9ab7e8..87cee18cd18f 100644 --- a/apps/hannk/util/buffer_util.h +++ b/apps/hannk/util/buffer_util.h @@ -36,11 +36,11 @@ template class Functor, typename... Args> auto dynamic_type_dispatch(const halide_type_t &type, Args &&...args) -> decltype(std::declval>()(std::forward(args)...)) { -#define HANDLE_CASE(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): \ +#define HANDLE_CASE(CODE, BITS, TYPE) \ + case halide_type_t(CODE, BITS): \ return Functor()(std::forward(args)...); - switch (type.element_of().as_u32()) { + switch (type) { // HANDLE_CASE(halide_type_float, 16, float) // TODO HANDLE_CASE(halide_type_float, 32, float) HANDLE_CASE(halide_type_float, 64, double) diff --git a/apps/hannk/util/error_util.cpp b/apps/hannk/util/error_util.cpp index e9e21186b18c..7a8abf677800 100644 --- a/apps/hannk/util/error_util.cpp +++ b/apps/hannk/util/error_util.cpp @@ -4,20 +4,6 @@ namespace hannk { -std::ostream &operator<<(std::ostream &stream, const halide_type_t &type) { - if (type.code == halide_type_uint && type.bits == 1) { - stream << "bool"; - } else { - static const char *const names[5] = {"int", "uint", "float", "handle", "bfloat"}; - assert(type.code >= 0 && type.code < size(names)); - stream << names[type.code] << (int)type.bits; - } - if (type.lanes > 1) { - stream << "x" << (int)type.lanes; - } - return stream; -} - std::ostream &operator<<(std::ostream &s, const halide_dimension_t &dim) { return s << "{" << dim.min << ", " << dim.extent << ", " << dim.stride << "}"; } diff --git a/apps/hannk/util/error_util.h b/apps/hannk/util/error_util.h index fb28114eab2b..340a3421d031 100644 --- a/apps/hannk/util/error_util.h +++ b/apps/hannk/util/error_util.h @@ -16,7 +16,6 @@ constexpr size_t size(T (&)[N]) { return N; } -std::ostream &operator<<(std::ostream &stream, const halide_type_t &type); std::ostream &operator<<(std::ostream &s, const halide_dimension_t &dim); template diff --git a/python_bindings/src/halide/halide_/PyBuffer.h b/python_bindings/src/halide/halide_/PyBuffer.h index 20bf33823a41..64858213020e 100644 --- a/python_bindings/src/halide/halide_/PyBuffer.h +++ b/python_bindings/src/halide/halide_/PyBuffer.h @@ -32,7 +32,7 @@ Halide::Runtime::Buffer pybufferinfo_to_halidebuffer const int dst_axis = reverse_axes ? (info.ndim - i - 1) : i; dims[dst_axis] = {0, (int32_t)info.shape[i], (int32_t)elem_stride}; } - return Halide::Runtime::Buffer(t, info.ptr, (int)info.ndim, dims); + return Halide::Runtime::Buffer(t.to_abi(), info.ptr, (int)info.ndim, dims); } } // namespace PythonBindings diff --git a/python_bindings/src/halide/halide_/PyCallable.cpp b/python_bindings/src/halide/halide_/PyCallable.cpp index 76bf34d6d240..dd3122ad0ee4 100644 --- a/python_bindings/src/halide/halide_/PyCallable.cpp +++ b/python_bindings/src/halide/halide_/PyCallable.cpp @@ -118,12 +118,12 @@ class PyCallable { argv[slot] = &scalar_storage[slot]; #define HALIDE_HANDLE_TYPE_DISPATCH(CODE, BITS, TYPE, FIELD) \ - case halide_type_t(CODE, BITS).as_u32(): \ + case halide_type_t(CODE, BITS): \ scalar_storage[slot].u.FIELD = cast_to(value); \ cci[slot] = Callable::make_scalar_qcci(halide_type_t(CODE, BITS)); \ break; - switch (((halide_type_t)c_arg.type).element_of().as_u32()) { + switch (c_arg.type.to_abi()) { HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 32, float, f32) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 64, double, f64) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_int, 8, int8_t, i8) @@ -137,7 +137,7 @@ class PyCallable { HALIDE_HANDLE_TYPE_DISPATCH(halide_type_uint, 64, uint64_t, u64) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_handle, 64, uint64_t, u64) // Handle types are always uint64, regardless of pointer size default: - _halide_user_assert(0) << "Unsupported type in Callable argument list: " << c_arg.type << "\n"; + _halide_user_error << "Unsupported type in Callable argument list: " << c_arg.type << "\n"; } #undef HALIDE_HANDLE_TYPE_DISPATCH diff --git a/python_bindings/src/halide/halide_/PyType.cpp b/python_bindings/src/halide/halide_/PyType.cpp index 08847dad852b..fd79ed3cbf5a 100644 --- a/python_bindings/src/halide/halide_/PyType.cpp +++ b/python_bindings/src/halide/halide_/PyType.cpp @@ -36,8 +36,8 @@ std::string halide_type_to_string(const Type &type) { stream << "bfloat"; break; case halide_type_handle: - if (type.handle_type) { - stream << type.handle_type->inner_name.name; + if (type.handle_type()) { + stream << type.handle_type()->inner_name.name; } else { stream << "handle"; } diff --git a/src/AddImageChecks.cpp b/src/AddImageChecks.cpp index eba08b820d14..4b9312854dd4 100644 --- a/src/AddImageChecks.cpp +++ b/src/AddImageChecks.cpp @@ -330,7 +330,7 @@ Stmt add_image_checks_inner(Stmt s, { string type_name = name + ".type"; Expr type_var = Variable::make(UInt(32), type_name, image, param, rdom); - uint32_t correct_type_bits = ((halide_type_t)type).as_u32(); + uint32_t correct_type_bits = type.to_abi(); Expr correct_type_expr = make_const(UInt(32), correct_type_bits); Expr error = Call::make(Int(32), "halide_error_bad_type", {error_name, type_var, correct_type_expr}, diff --git a/src/Buffer.h b/src/Buffer.h index f1a610eb90fb..ee06fd916252 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -210,7 +210,7 @@ class Buffer { typename = std::enable_if_t::value>> explicit Buffer(Type t, int first, Args... rest) - : Buffer(Runtime::Buffer(t, Internal::get_shape_from_start_of_parameter_pack(first, rest...)), + : Buffer(Runtime::Buffer(t.to_abi(), Internal::get_shape_from_start_of_parameter_pack(first, rest...)), Internal::get_name_from_end_of_parameter_pack(rest...)) { } @@ -229,14 +229,14 @@ class Buffer { explicit Buffer(Type t, const std::vector &sizes, const std::string &name = "") - : Buffer(Runtime::Buffer(t, sizes), name) { + : Buffer(Runtime::Buffer(t.to_abi(), sizes), name) { } explicit Buffer(Type t, const std::vector &sizes, const std::vector &storage_order, const std::string &name = "") - : Buffer(Runtime::Buffer(t, sizes, storage_order), name) { + : Buffer(Runtime::Buffer(t.to_abi(), sizes, storage_order), name) { } explicit Buffer(const std::vector &sizes, @@ -261,7 +261,7 @@ class Buffer { explicit Buffer(Type t, Internal::add_const_if_T_is_const *data, int first, Args &&...rest) - : Buffer(Runtime::Buffer(t, data, Internal::get_shape_from_start_of_parameter_pack(first, rest...)), + : Buffer(Runtime::Buffer(t.to_abi(), data, Internal::get_shape_from_start_of_parameter_pack(first, rest...)), Internal::get_name_from_end_of_parameter_pack(rest...)) { } @@ -271,7 +271,7 @@ class Buffer { Internal::add_const_if_T_is_const *data, const std::vector &sizes, const std::string &name = "") - : Buffer(Runtime::Buffer(t, data, sizes, name)) { + : Buffer(Runtime::Buffer(t.to_abi(), data, sizes, name)) { } template *data, const std::vector &sizes, const std::string &name = "") - : Buffer(Runtime::Buffer(t, data, sizes), name) { + : Buffer(Runtime::Buffer(t.to_abi(), data, sizes), name) { } explicit Buffer(Type t, @@ -300,7 +300,7 @@ class Buffer { int d, const halide_dimension_t *shape, const std::string &name = "") - : Buffer(Runtime::Buffer(t, data, d, shape), name) { + : Buffer(Runtime::Buffer(t.to_abi(), data, d, shape), name) { } explicit Buffer(T *data, @@ -315,7 +315,7 @@ class Buffer { } static Buffer<> make_scalar(Type t, const std::string &name = "") { - return Buffer<>(Runtime::Buffer<>::make_scalar(t), name); + return Buffer<>(Runtime::Buffer<>::make_scalar(t.to_abi()), name); } static Buffer make_scalar(T *data, const std::string &name = "") { @@ -327,7 +327,7 @@ class Buffer { } static Buffer<> make_interleaved(Type t, int width, int height, int channels, const std::string &name = "") { - return Buffer<>(Runtime::Buffer<>::make_interleaved(t, width, height, channels), name); + return Buffer<>(Runtime::Buffer<>::make_interleaved(t.to_abi(), width, height, channels), name); } static Buffer make_interleaved(T *data, int width, int height, int channels, const std::string &name = "") { @@ -337,7 +337,7 @@ class Buffer { static Buffer> make_interleaved(Type t, T *data, int width, int height, int channels, const std::string &name = "") { using T2 = Internal::add_const_if_T_is_const; - return Buffer(Runtime::Buffer::make_interleaved(t, data, width, height, channels), name); + return Buffer(Runtime::Buffer::make_interleaved(t.to_abi(), data, width, height, channels), name); } template diff --git a/src/CPlusPlusMangle.cpp b/src/CPlusPlusMangle.cpp index 6beb9e83f6c1..ea419c08f26f 100644 --- a/src/CPlusPlusMangle.cpp +++ b/src/CPlusPlusMangle.cpp @@ -150,13 +150,13 @@ struct QualsState { if (finished || (!is_pointer && !last_is_pointer && - type.handle_type->reference_type == halide_handle_cplusplus_type::NotReference)) { + type.handle_type()->reference_type == halide_handle_cplusplus_type::NotReference)) { finished = true; return; } result = one_qualifier_set(last_is_const, last_is_volatile, last_is_restrict, last_is_pointer, base_mode) + result; - if (last_is_pointer && (is_pointer || type.handle_type->reference_type != halide_handle_cplusplus_type::NotReference)) { + if (last_is_pointer && (is_pointer || type.handle_type()->reference_type != halide_handle_cplusplus_type::NotReference)) { result = one_qualifier_set(last_is_const, last_is_volatile, last_is_restrict, false, base_mode) + result; } @@ -174,9 +174,9 @@ struct QualsState { result = one_qualifier_set(false, false, false, last_is_pointer, base_mode) + result; } - if (type.handle_type->reference_type == halide_handle_cplusplus_type::LValueReference) { + if (type.handle_type()->reference_type == halide_handle_cplusplus_type::LValueReference) { result = "A" + base_mode + result; // Or is it "R"? - } else if (type.handle_type->reference_type == halide_handle_cplusplus_type::RValueReference) { + } else if (type.handle_type()->reference_type == halide_handle_cplusplus_type::RValueReference) { result = "$$Q" + base_mode + result; } } @@ -188,7 +188,7 @@ struct QualsState { std::string mangle_indirection_and_cvr_quals(const Type &type, const Target &target) { QualsState state(type, (target.bits == 64) ? "E" : ""); - for (uint8_t modifier : type.handle_type->cpp_type_modifiers) { + for (uint8_t modifier : type.handle_type()->cpp_type_modifiers) { state.handle_modifier(modifier); } state.final(); @@ -200,28 +200,28 @@ MangledNamePart mangle_inner_name(const Type &type, const Target &target, Previo MangledNamePart result(""); std::string quals = mangle_indirection_and_cvr_quals(type, target); - if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { - return quals + simple_type_to_mangle_char(type.handle_type->inner_name.name, target); + if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { + return quals + simple_type_to_mangle_char(type.handle_type()->inner_name.name, target); } else { std::string code; - if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Struct) { + if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Struct) { code = "U"; - } else if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Class) { + } else if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Class) { code = "V"; - } else if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Union) { + } else if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Union) { code = "T"; - } else if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Enum) { + } else if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Enum) { code = "W4"; } - result.full_name = quals + code + type.handle_type->inner_name.name + "@"; - result.with_substitutions = quals + code + prev_decls.check_and_enter_name(type.handle_type->inner_name.name); + result.full_name = quals + code + type.handle_type()->inner_name.name + "@"; + result.with_substitutions = quals + code + prev_decls.check_and_enter_name(type.handle_type()->inner_name.name); - for (const auto &enclosing_type : reverse_view(type.handle_type->enclosing_types)) { + for (const auto &enclosing_type : reverse_view(type.handle_type()->enclosing_types)) { result.full_name += enclosing_type.name + "@"; result.with_substitutions += prev_decls.check_and_enter_name(enclosing_type.name); } - for (const auto &ns : reverse_view(type.handle_type->namespaces)) { + for (const auto &ns : reverse_view(type.handle_type()->namespaces)) { result.full_name += ns + "@"; result.with_substitutions += prev_decls.check_and_enter_name(ns); } @@ -273,7 +273,7 @@ MangledNamePart mangle_type(const Type &type, const Target &target, PreviousDecl internal_error << "Unexpected floating-point type size: " << type.bits() << ".\n"; return ""; } else if (type.is_handle()) { - return mangle_inner_name((type.handle_type != nullptr) ? type : non_null_void_star_type(), + return mangle_inner_name((type.handle_type() != nullptr) ? type : non_null_void_star_type(), target, prev_decls); } internal_error << "Unexpected kind of type. Code: " << type.code() << "\n"; @@ -427,11 +427,11 @@ struct PrevPrefixes { MangledNamePart apply_indirection_and_cvr_quals(const Type &type, MangledNamePart &name_part, PrevPrefixes &prevs) { - for (uint8_t modifier : type.handle_type->cpp_type_modifiers) { + for (uint8_t modifier : type.handle_type()->cpp_type_modifiers) { // Qualifiers on a value type are simply not encoded. // E.g. "int f(const int)" mangles the same as "int f(int)". if (!(modifier & halide_handle_cplusplus_type::Pointer) && - type.handle_type->reference_type == halide_handle_cplusplus_type::NotReference) { + type.handle_type()->reference_type == halide_handle_cplusplus_type::NotReference) { break; } @@ -458,9 +458,9 @@ MangledNamePart apply_indirection_and_cvr_quals(const Type &type, MangledNamePar } } - if (type.handle_type->reference_type == halide_handle_cplusplus_type::LValueReference) { + if (type.handle_type()->reference_type == halide_handle_cplusplus_type::LValueReference) { prevs.prepend_name_part("R", name_part); - } else if (type.handle_type->reference_type == halide_handle_cplusplus_type::RValueReference) { + } else if (type.handle_type()->reference_type == halide_handle_cplusplus_type::RValueReference) { prevs.prepend_name_part("O", name_part); } @@ -515,12 +515,12 @@ MangledNamePart mangle_qualified_name(const std::string &name, const std::vector } std::string mangle_inner_name(const Type &type, const Target &target, PrevPrefixes &prevs) { - if (type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { - MangledNamePart result = simple_type_to_mangle_char(type.handle_type->inner_name.name, target); + if (type.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { + MangledNamePart result = simple_type_to_mangle_char(type.handle_type()->inner_name.name, target); return apply_indirection_and_cvr_quals(type, result, prevs).full_name; } else { - MangledNamePart mangled = mangle_qualified_name(type.handle_type->inner_name.name, type.handle_type->namespaces, - type.handle_type->enclosing_types, true, prevs); + MangledNamePart mangled = mangle_qualified_name(type.handle_type()->inner_name.name, type.handle_type()->namespaces, + type.handle_type()->enclosing_types, true, prevs); return apply_indirection_and_cvr_quals(type, mangled, prevs).full_name; } } @@ -585,7 +585,7 @@ std::string mangle_type(const Type &type, const Target &target, PrevPrefixes &pr internal_error << "Unexpected floating-point type size: " << type.bits() << ".\n"; return ""; } else if (type.is_handle()) { - return mangle_inner_name((type.handle_type != nullptr) ? type : non_null_void_star_type(), + return mangle_inner_name((type.handle_type() != nullptr) ? type : non_null_void_star_type(), target, prevs); } internal_error << "Unexpected kind of type. Code: " << type.code() << "\n"; diff --git a/src/Callable.cpp b/src/Callable.cpp index 95a34ed455b1..0ffc6dd2c080 100644 --- a/src/Callable.cpp +++ b/src/Callable.cpp @@ -68,7 +68,7 @@ Callable::Callable(const std::string &name, for (const Argument &a : contents->jit_cache.arguments) { const auto qcci = (a.name == "__user_context") ? Callable::make_ucon_qcci() : - (a.is_scalar() ? Callable::make_scalar_qcci(a.type) : Callable::make_buffer_qcci()); + (a.is_scalar() ? Callable::make_scalar_qcci(a.type.to_abi()) : Callable::make_buffer_qcci()); contents->quick_call_check_info.push_back(qcci); } @@ -146,7 +146,7 @@ Callable::FailureFn Callable::check_fcci(size_t argc, const FullCallCheckInfo *a if (contents->full_call_check_info.empty()) { contents->full_call_check_info.reserve(contents->jit_cache.arguments.size()); for (const Argument &a : contents->jit_cache.arguments) { - const auto fcci = a.is_scalar() ? Callable::make_scalar_fcci(a.type) : Callable::make_buffer_fcci(a.type, a.dimensions); + const auto fcci = a.is_scalar() ? Callable::make_scalar_fcci(a.type.to_abi()) : Callable::make_buffer_fcci(a.type.to_abi(), a.dimensions); contents->full_call_check_info.push_back(fcci); } } diff --git a/src/Callable.h b/src/Callable.h index acdfa77a2e3a..6057e1a087c6 100644 --- a/src/Callable.h +++ b/src/Callable.h @@ -140,24 +140,33 @@ class Callable { // --------------------------------- - // This value is constructed so we can do a complete type-and-dim check - // of Buffers, and is used for the make_std_function() method, to ensure - // that if we specify static type-and-dims for Buffers, the ones we specify - // actually match the underlying code. We take horrible liberties with halide_type_t - // to make this happen -- specifically, encoding dimensionality and buffer-vs-scalar - // into the 'lanes' field -- but that's ok since this never escapes into other usage. - using FullCallCheckInfo = halide_type_t; - - static constexpr FullCallCheckInfo _make_fcci(halide_type_t type, int dims, bool is_buffer) { - return type.with_lanes(((uint16_t)dims << 1) | (uint16_t)(is_buffer ? 1 : 0)); - } + // This struct is used to do a complete type-and-dim check of Buffers, and + // is used for the make_std_function() method, to ensure that if we specify + // static type-and-dims for Buffers, the ones we specify actually match the + // underlying code. Unlike QuickCallCheckInfo, this needs to be able to + // represent "type unspecified" and "dims unspecified" (e.g. for + // Buffer args). + struct FullCallCheckInfo { + // The scalar ABI element type; code == 0 && bits == 0 means "unspecified". + halide_type_t type; + // Buffer dimensionality; negative means "unspecified". Unused for scalars. + int8_t dims; + bool is_buffer; + + constexpr bool operator==(const FullCallCheckInfo &other) const { + return is_buffer == other.is_buffer && dims == other.dims && type == other.type; + } + constexpr bool operator!=(const FullCallCheckInfo &other) const { + return !(*this == other); + } + }; static constexpr FullCallCheckInfo make_scalar_fcci(halide_type_t t) { - return _make_fcci(t, 0, false); + return FullCallCheckInfo{t, 0, false}; } static constexpr FullCallCheckInfo make_buffer_fcci(halide_type_t t, int dims) { - return _make_fcci(t, dims, true); + return FullCallCheckInfo{t, (int8_t)dims, true}; } static bool is_compatible_fcci(FullCallCheckInfo actual, FullCallCheckInfo expected) { @@ -165,24 +174,19 @@ class Callable { return true; // my, that was easy } - // Might still be compatible - const bool a_is_buffer = (actual.lanes & 1) != 0; - const int a_dims = (((int16_t)actual.lanes) >> 1); - const halide_type_t a_type = actual.with_lanes(0); - - const bool e_is_buffer = (expected.lanes & 1) != 0; - const int e_dims = (((int16_t)expected.lanes) >> 1); - const halide_type_t e_type = expected.with_lanes(0); + // The "no type specified" sentinel: code/bits both zero. + constexpr halide_type_t none{(halide_type_code_t)0, 0}; - const bool types_match = (a_type == halide_type_t()) || - (e_type == halide_type_t()) || - (a_type == e_type); + // Might still be compatible + const bool types_match = actual.type == none || + expected.type == none || + actual.type == expected.type; - const bool dims_match = a_dims < 0 || - e_dims < 0 || - a_dims == e_dims; + const bool dims_match = actual.dims < 0 || + expected.dims < 0 || + actual.dims == expected.dims; - return a_is_buffer == e_is_buffer && types_match && dims_match; + return actual.is_buffer == expected.is_buffer && types_match && dims_match; } template diff --git a/src/CodeGen_ARM.cpp b/src/CodeGen_ARM.cpp index dbc8458a999e..e4d2c5b67624 100644 --- a/src/CodeGen_ARM.cpp +++ b/src/CodeGen_ARM.cpp @@ -432,9 +432,9 @@ constexpr int max_intrinsic_args = 4; struct ArmIntrinsic { const char *arm32; const char *arm64; - halide_type_t ret_type; + Type ret_type; const char *name; - halide_type_t arg_types[max_intrinsic_args]; + Type arg_types[max_intrinsic_args]; int flags; enum { AllowUnsignedOp1 = 1 << 0, // Generate a second version of the instruction with the second operand unsigned. @@ -1156,7 +1156,7 @@ void CodeGen_ARM::init_module() { }(); int width_factor = 1; - if (!((intrin.ret_type.lanes <= 1) && (intrin.flags & ArmIntrinsic::NoMangle))) { + if (!((intrin.ret_type.lanes() <= 1) && (intrin.flags & ArmIntrinsic::NoMangle))) { switch (flavor) { case SIMDFlavors::NeonWidthX1: width_factor = 1; @@ -1181,8 +1181,8 @@ void CodeGen_ARM::init_module() { } vector arg_types; arg_types.reserve(4); - for (halide_type_t i : intrin.arg_types) { - if (i.bits == 0) { + for (const Type &i : intrin.arg_types) { + if (i.bits() == 0) { break; } Type arg_type = i; diff --git a/src/CodeGen_C.cpp b/src/CodeGen_C.cpp index 7b5a9792247e..c4de541f20b5 100644 --- a/src/CodeGen_C.cpp +++ b/src/CodeGen_C.cpp @@ -252,8 +252,8 @@ CodeGen_C::CodeGen_C(ostream &s, const Target &t, OutputKind output_kind, const << "struct halide_filter_metadata_t;\n" << "\n"; // We just forward declared the following types: - forward_declared.insert(type_of().handle_type); - forward_declared.insert(type_of().handle_type); + forward_declared.insert(type_of().handle_type()); + forward_declared.insert(type_of().handle_type()); } else if (is_extern_decl()) { // Extern decls to be wrapped inside other code (eg python extensions); // emit the forward decls with a minimum of noise. Note that we never @@ -261,8 +261,8 @@ CodeGen_C::CodeGen_C(ostream &s, const Target &t, OutputKind output_kind, const stream << "struct halide_buffer_t;\n" << "struct halide_filter_metadata_t;\n" << "\n"; - forward_declared.insert(type_of().handle_type); - forward_declared.insert(type_of().handle_type); + forward_declared.insert(type_of().handle_type()); + forward_declared.insert(type_of().handle_type()); } else { // Include declarations of everything generated C source might want stream @@ -569,37 +569,37 @@ class ExternCallPrototypes : public IRGraphVisitor { } // namespace void CodeGen_C::forward_declare_type_if_needed(const Type &t) { - if (!t.handle_type || - forward_declared.count(t.handle_type) || - t.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { + if (!t.handle_type() || + forward_declared.count(t.handle_type()) || + t.handle_type()->inner_name.cpp_type_type == halide_cplusplus_type_name::Simple) { return; } - for (const auto &ns : t.handle_type->namespaces) { + for (const auto &ns : t.handle_type()->namespaces) { stream << "namespace " << ns << " { "; } - switch (t.handle_type->inner_name.cpp_type_type) { + switch (t.handle_type()->inner_name.cpp_type_type) { case halide_cplusplus_type_name::Simple: // nothing break; case halide_cplusplus_type_name::Struct: - stream << "struct " << t.handle_type->inner_name.name << ";"; + stream << "struct " << t.handle_type()->inner_name.name << ";"; break; case halide_cplusplus_type_name::Class: - stream << "class " << t.handle_type->inner_name.name << ";"; + stream << "class " << t.handle_type()->inner_name.name << ";"; break; case halide_cplusplus_type_name::Union: - stream << "union " << t.handle_type->inner_name.name << ";"; + stream << "union " << t.handle_type()->inner_name.name << ";"; break; case halide_cplusplus_type_name::Enum: internal_error << "Passing pointers to enums is unsupported\n"; break; } - for (const auto &ns : t.handle_type->namespaces) { + for (const auto &ns : t.handle_type()->namespaces) { (void)ns; stream << " }"; } stream << "\n"; - forward_declared.insert(t.handle_type); + forward_declared.insert(t.handle_type()); } void CodeGen_C::emit_argv_wrapper(const std::string &function_name, @@ -811,7 +811,7 @@ void CodeGen_C::emit_metadata_getter(const std::string &function_name, stream << get_indent() << kind_names[arg.kind] << ",\n"; stream << get_indent() << (int)arg.dimensions << ",\n"; internal_assert(arg.type.code() < sizeof(type_code_names) / sizeof(type_code_names[0])); - stream << get_indent() << "{" << type_code_names[arg.type.code()] << ", " << arg.type.bits() << ", " << arg.type.lanes() << "},\n"; + stream << get_indent() << "{" << type_code_names[arg.type.code()] << ", " << arg.type.bits() << "},\n"; stream << get_indent() << "scalar_def_" << legalized_name << ",\n"; stream << get_indent() << "scalar_min_" << legalized_name << ",\n"; stream << get_indent() << "scalar_max_" << legalized_name << ",\n"; @@ -888,7 +888,7 @@ void CodeGen_C::emit_constexpr_function_info(const std::string &function_name, stream << get_indent() << "{\"" << name << "\", " << kind_names[arg.kind] << ", " << (int)arg.dimensions << ", halide_type_t{" << type_code_names[arg.type.code()] << ", " << arg.type.bits() - << ", " << arg.type.lanes() << "}},\n"; + << "}},\n"; } indent -= 1; stream << get_indent() << "}};\n"; @@ -1230,11 +1230,11 @@ void CodeGen_C::compile(const Buffer<> &buffer) { // case the buffer objects need to be non-const, because the constness // (from the POV of the extern stage) is a runtime property. stream << "static halide_buffer_t " << name << "_buffer_ = {" - << "0, " // device - << "nullptr, " // device_interface - << "const_cast(&" << name << "_data[0]), " // host - << "0, " // flags - << "halide_type_t((halide_type_code_t)(" << (int)t.code() << "), " << t.bits() << ", " << t.lanes() << "), " + << "0, " // device + << "nullptr, " // device_interface + << "const_cast(&" << name << "_data[0]), " // host + << "0, " // flags + << "halide_type_t((halide_type_code_t)(" << (int)t.code() << "), " << t.bits() << "), " // scalar element type << buffer.dimensions() << ", " << buffer_shape << "};\n"; @@ -1660,7 +1660,7 @@ void CodeGen_C::visit(const Call *op) { } } else if (op->is_intrinsic(Call::make_struct)) { if (op->args.empty()) { - internal_assert(op->type.handle_type); + internal_assert(op->type.handle_type()); // Add explicit cast so that different structs can't cache to the same value rhs << "(" << print_type(op->type) << ")(nullptr)"; } else if (op->type == type_of()) { @@ -1736,7 +1736,7 @@ void CodeGen_C::visit(const Call *op) { // TODO: This is dubious type-punning. We really need to // find a better way to do this. We dodge the problem for // the specific case of buffer shapes in the case above. - if (op->type.handle_type) { + if (op->type.handle_type()) { rhs << "(" << print_type(op->type) << ")"; } rhs << struct_name; @@ -1759,7 +1759,7 @@ void CodeGen_C::visit(const Call *op) { // TODO: This is dubious type-punning. We really need to // find a better way to do this. We dodge the problem for // the specific case of buffer shapes in the case above. - if (op->type.handle_type) { + if (op->type.handle_type()) { rhs << "(" << print_type(op->type) << ")"; } rhs << "(&" << struct_name << ")"; @@ -2730,10 +2730,10 @@ int test1(struct halide_buffer_t *_buf_buffer, float _alpha, int32_t _beta, void inline constexpr std::array<::HalideFunctionInfo::ArgumentInfo, 4> test1_argument_info() { return {{ - {"buf", ::HalideFunctionInfo::OutputBuffer, 3, halide_type_t{halide_type_int, 32, 1}}, - {"alpha", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_float, 32, 1}}, - {"beta", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_int, 32, 1}}, - {"__user_context", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_handle, 64, 1}}, + {"buf", ::HalideFunctionInfo::OutputBuffer, 3, halide_type_t{halide_type_int, 32}}, + {"alpha", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_float, 32}}, + {"beta", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_int, 32}}, + {"__user_context", ::HalideFunctionInfo::InputScalar, 0, halide_type_t{halide_type_handle, 64}}, }}; } #endif diff --git a/src/CodeGen_Hexagon.cpp b/src/CodeGen_Hexagon.cpp index c680fdb1a82e..c60744e76ca7 100644 --- a/src/CodeGen_Hexagon.cpp +++ b/src/CodeGen_Hexagon.cpp @@ -543,23 +543,23 @@ struct HvxIntrinsic { v65OrLater = 1 << 1, }; llvm::Intrinsic::ID id; - halide_type_t ret_type; + Type ret_type; const char *name; - halide_type_t arg_types[4]; + Type arg_types[4]; int flags; }; // TODO: these should probably be declared constexpr, but that would -// require marking various halide_type_t methods as constexpr, and an +// require marking various Type methods as constexpr, and an // obscure bug in MSVC2017 can cause compilation failures for them. // The bug appears to be fixed in MSVC2019, so when we move to that // as a baseline for Windows, this should be revisited. -halide_type_t i8 = halide_type_t(halide_type_int, 8); -halide_type_t i16 = halide_type_t(halide_type_int, 16); -halide_type_t i32 = halide_type_t(halide_type_int, 32); -halide_type_t u8 = halide_type_t(halide_type_uint, 8); -halide_type_t u16 = halide_type_t(halide_type_uint, 16); -halide_type_t u32 = halide_type_t(halide_type_uint, 32); +Type i8 = Int(8); +Type i16 = Int(16); +Type i32 = Int(32); +Type u8 = UInt(8); +Type u16 = UInt(16); +Type u32 = UInt(32); // Define vectors that are 1x and 2x the Hexagon HVX width -- // Note that we use placeholders here (which we fix up when processing @@ -568,19 +568,19 @@ halide_type_t u32 = halide_type_t(halide_type_uint, 32); // data, rather than having to assemble it at runtime. constexpr int kOneX = 64 * 8; -halide_type_t i8v1 = i8.with_lanes(kOneX / 8); -halide_type_t i16v1 = i16.with_lanes(kOneX / 16); -halide_type_t i32v1 = i32.with_lanes(kOneX / 32); -halide_type_t u8v1 = u8.with_lanes(kOneX / 8); -halide_type_t u16v1 = u16.with_lanes(kOneX / 16); -halide_type_t u32v1 = u32.with_lanes(kOneX / 32); +Type i8v1 = i8.with_lanes(kOneX / 8); +Type i16v1 = i16.with_lanes(kOneX / 16); +Type i32v1 = i32.with_lanes(kOneX / 32); +Type u8v1 = u8.with_lanes(kOneX / 8); +Type u16v1 = u16.with_lanes(kOneX / 16); +Type u32v1 = u32.with_lanes(kOneX / 32); -halide_type_t i8v2 = i8v1.with_lanes(i8v1.lanes * 2); -halide_type_t i16v2 = i16v1.with_lanes(i16v1.lanes * 2); -halide_type_t i32v2 = i32v1.with_lanes(i32v1.lanes * 2); -halide_type_t u8v2 = u8v1.with_lanes(u8v1.lanes * 2); -halide_type_t u16v2 = u16v1.with_lanes(u16v1.lanes * 2); -halide_type_t u32v2 = u32v1.with_lanes(u32v1.lanes * 2); +Type i8v2 = i8v1.with_lanes(i8v1.lanes() * 2); +Type i16v2 = i16v1.with_lanes(i16v1.lanes() * 2); +Type i32v2 = i32v1.with_lanes(i32v1.lanes() * 2); +Type u8v2 = u8v1.with_lanes(u8v1.lanes() * 2); +Type u16v2 = u16v1.with_lanes(u16v1.lanes() * 2); +Type u32v2 = u32v1.with_lanes(u32v1.lanes() * 2); #define INTRINSIC_128B(id) llvm::Intrinsic::hexagon_V6_##id##_128B const HvxIntrinsic intrinsic_wrappers[] = { @@ -853,11 +853,11 @@ void CodeGen_Hexagon::init_module() { // operands, they all operate on vectors of 32 bit integers. To make // it easier to generate code, we define wrapper intrinsics with // the correct type (plus the necessary bitcasts). - const auto fix_lanes = [&](const halide_type_t &t) -> halide_type_t { - if (t.lanes == 1) { + const auto fix_lanes = [&](const Type &t) -> Type { + if (t.lanes() == 1) { return t; } - const int lanes_actual = ((int)t.lanes * native_vector_bits()) / kOneX; + const int lanes_actual = ((int)t.lanes() * native_vector_bits()) / kOneX; return t.with_lanes(lanes_actual); }; @@ -867,10 +867,10 @@ void CodeGen_Hexagon::init_module() { internal_assert(id != llvm::Intrinsic::not_intrinsic); // Get the real intrinsic. llvm::Function *intrin = llvm::Intrinsic::getOrInsertDeclaration(module.get(), id); - halide_type_t ret_type = fix_lanes(i.ret_type); + Type ret_type = fix_lanes(i.ret_type); arg_types.clear(); for (const auto &a : i.arg_types) { - if (a.bits == 0) { + if (a.bits() == 0) { break; } arg_types.emplace_back(fix_lanes(a)); diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp index 9263aef95f75..78a0567161ed 100644 --- a/src/CodeGen_LLVM.cpp +++ b/src/CodeGen_LLVM.cpp @@ -841,10 +841,16 @@ void CodeGen_LLVM::compile_buffer(const Buffer<> &buf) { << "Can't embed Image \"" << buf.name() << "\"" << " because it has a dirty device pointer\n"; + // The ABI halide_type_t is a scalar element type: {code, bits, reserved}. + // A buffer's element type is always scalar, and the wire struct does not + // carry lanes (the third field is now reserved and must be zero). + internal_assert(buf.type().lanes() == 1) + << "Embedded buffer " << buf.name() + << " has a non-scalar element type with " << buf.type().lanes() << " lanes.\n"; Constant *type_fields[] = { ConstantInt::get(i8_t, buf.type().code()), ConstantInt::get(i8_t, buf.type().bits()), - ConstantInt::get(i16_t, buf.type().lanes())}; + ConstantInt::get(i16_t, 0)}; Constant *shape = nullptr; if (buf.dimensions()) { @@ -1039,7 +1045,7 @@ llvm::Function *CodeGen_LLVM::embed_metadata_getter(const std::string &metadata_ Constant *type_fields[] = { ConstantInt::get(i8_t, args[arg].type.code()), ConstantInt::get(i8_t, args[arg].type.bits()), - ConstantInt::get(i16_t, 1)}; + ConstantInt::get(i16_t, 0)}; // reserved (formerly lanes); must be 0 Constant *type = ConstantStruct::get(type_t_type, type_fields); auto argument_estimates = args[arg].argument_estimates; diff --git a/src/CodeGen_PowerPC.cpp b/src/CodeGen_PowerPC.cpp index 2dd44487af7f..d8bebeebb7fa 100644 --- a/src/CodeGen_PowerPC.cpp +++ b/src/CodeGen_PowerPC.cpp @@ -45,9 +45,9 @@ const int max_intrinsic_args = 4; struct PowerPCIntrinsic { const char *intrin_name; - halide_type_t ret_type; + Type ret_type; const char *name; - halide_type_t arg_types[max_intrinsic_args]; + Type arg_types[max_intrinsic_args]; Target::Feature feature = Target::FeatureEnd; }; @@ -107,8 +107,8 @@ void CodeGen_PowerPC::init_module() { Type ret_type = i.ret_type; vector arg_types; arg_types.reserve(max_intrinsic_args); - for (halide_type_t j : i.arg_types) { - if (j.bits == 0) { + for (const Type &j : i.arg_types) { + if (j.bits() == 0) { break; } arg_types.emplace_back(j); diff --git a/src/CodeGen_WebAssembly.cpp b/src/CodeGen_WebAssembly.cpp index 5cf9d1744181..5047fa644c09 100644 --- a/src/CodeGen_WebAssembly.cpp +++ b/src/CodeGen_WebAssembly.cpp @@ -54,9 +54,9 @@ constexpr int max_intrinsic_args = 4; struct WasmIntrinsic { const char *intrin_name; - halide_type_t ret_type; + Type ret_type; const char *name; - halide_type_t arg_types[max_intrinsic_args]; + Type arg_types[max_intrinsic_args]; Target::Feature feature = Target::FeatureEnd; }; @@ -124,8 +124,8 @@ void CodeGen_WebAssembly::init_module() { Type ret_type = i.ret_type; vector arg_types; arg_types.reserve(max_intrinsic_args); - for (halide_type_t i : i.arg_types) { - if (i.bits == 0) { + for (const Type &i : i.arg_types) { + if (i.bits() == 0) { break; } arg_types.emplace_back(i); diff --git a/src/CodeGen_X86.cpp b/src/CodeGen_X86.cpp index 4ff685a7a4e2..6c479193a2c3 100644 --- a/src/CodeGen_X86.cpp +++ b/src/CodeGen_X86.cpp @@ -128,9 +128,9 @@ const int max_intrinsic_args = 6; struct x86Intrinsic { const char *intrin_name; - halide_type_t ret_type; + Type ret_type; const char *name; - halide_type_t arg_types[max_intrinsic_args]; + Type arg_types[max_intrinsic_args]; Target::Feature feature = Target::FeatureEnd; uint32_t flags = 0; enum Options { @@ -321,8 +321,8 @@ void CodeGen_X86::init_module() { Type ret_type = i.ret_type; vector arg_types; arg_types.reserve(max_intrinsic_args); - for (halide_type_t j : i.arg_types) { - if (j.bits == 0) { + for (const Type &j : i.arg_types) { + if (j.bits() == 0) { break; } arg_types.emplace_back(j); diff --git a/src/Deinterleave.cpp b/src/Deinterleave.cpp index e7ebb55bab2e..d552bc2ae8e6 100644 --- a/src/Deinterleave.cpp +++ b/src/Deinterleave.cpp @@ -709,13 +709,10 @@ class Interleaver : public IRMutator { return Stmt(); } - // Too many stores and lanes to represent in a single vector - // type. - int max_bits = sizeof(halide_type_t::lanes) * 8; + // Too many stores and lanes to represent in a single vector type. // mul_would_overflow is for signed types, but vector lanes // are unsigned, so add a bit. - max_bits++; - if (mul_would_overflow(max_bits, stores.size(), lanes)) { + if (mul_would_overflow(Type::kLanesBits + 1, stores.size(), lanes)) { return Stmt(); } diff --git a/src/IREquality.cpp b/src/IREquality.cpp index 1e7ae422c549..221e183a5db3 100644 --- a/src/IREquality.cpp +++ b/src/IREquality.cpp @@ -169,15 +169,13 @@ struct Comparer { HALIDE_ALWAYS_INLINE void cmp(const Type &a, const Type &b) { - uint32_t ta = ((halide_type_t)a).as_u32(); - uint32_t tb = ((halide_type_t)b).as_u32(); - if (ta < tb) { + if (a < b) { result = Order::LessThan; - } else if (ta > tb) { + } else if (b < a) { result = Order::GreaterThan; } else { - if (a.handle_type || b.handle_type) { - cmp(a.handle_type, b.handle_type); + if (a.handle_type() || b.handle_type()) { + cmp(a.handle_type(), b.handle_type()); } } } diff --git a/src/IRMatch.h b/src/IRMatch.h index d7d9b1281362..f0ec0c57a263 100644 --- a/src/IRMatch.h +++ b/src/IRMatch.h @@ -73,7 +73,36 @@ namespace IRMatcher { constexpr int max_wild = 6; -static const halide_type_t i64_type = {halide_type_int, 64, 1}; +static const Type i64_type = {halide_type_int, 64, 1}; + +/** The matcher only ever binds the types of *constants* (IntImm / UIntImm / + * FloatImm, possibly wrapped in a Broadcast), which are always numeric and thus + * never carry handle metadata. Storing them as full 8-byte `Type`s would double + * both the size of MatcherState and the per-node data movement during matching + * to carry a handle index that provably cannot be set. This 4-byte + * code+bits+lanes triple is full-fidelity for a constant's type — it preserves + * the broadcast lane count — at half the width, and restores the compact + * representation the matcher used before `Type` grew a handle-type index. + * Full `Type`s (with handles) still flow through the reconstruction path + * (`make(..., type_hint)`) and pattern-embedded types (e.g. CastOp), which is + * where handle fidelity actually matters. */ +struct BoundConstType { + halide_type_code_t code = halide_type_int; + uint8_t bits = 0; + uint16_t lanes = 0; + + HALIDE_ALWAYS_INLINE BoundConstType() noexcept = default; + HALIDE_ALWAYS_INLINE BoundConstType(Type t) noexcept + : code(t.code()), bits((uint8_t)t.bits()), lanes((uint16_t)t.lanes()) { + } + HALIDE_ALWAYS_INLINE operator Type() const noexcept { + return Type(code, bits, lanes); + } + HALIDE_ALWAYS_INLINE bool operator==(const BoundConstType &other) const noexcept { + return code == other.code && bits == other.bits && lanes == other.lanes; + } +}; +static_assert(sizeof(BoundConstType) == 4, "BoundConstType should be a compact 4-byte numeric type"); /** To save stack space, the matcher objects are largely stateless and * immutable. This state object is built up during matching and then @@ -83,7 +112,7 @@ struct MatcherState { const BaseExprNode *bindings[max_wild]; halide_scalar_value_t bound_const[max_wild]; - halide_type_t bound_const_type[max_wild]; + BoundConstType bound_const_type[max_wild]; HALIDE_ALWAYS_INLINE void set_binding(int i, const BaseExprNode &n) noexcept { @@ -96,60 +125,67 @@ struct MatcherState { } HALIDE_ALWAYS_INLINE - void set_bound_const(int i, int64_t s, halide_type_t t) noexcept { + void set_bound_const(int i, int64_t s, Type t) noexcept { bound_const[i].u.i64 = s; bound_const_type[i] = t; } HALIDE_ALWAYS_INLINE - void set_bound_const(int i, uint64_t u, halide_type_t t) noexcept { + void set_bound_const(int i, uint64_t u, Type t) noexcept { bound_const[i].u.u64 = u; bound_const_type[i] = t; } HALIDE_ALWAYS_INLINE - void set_bound_const(int i, double f, halide_type_t t) noexcept { + void set_bound_const(int i, double f, Type t) noexcept { bound_const[i].u.f64 = f; bound_const_type[i] = t; } HALIDE_ALWAYS_INLINE - void set_bound_const(int i, halide_scalar_value_t val, halide_type_t t) noexcept { + void set_bound_const(int i, halide_scalar_value_t val, Type t) noexcept { bound_const[i] = val; bound_const_type[i] = t; } HALIDE_ALWAYS_INLINE - void get_bound_const(int i, halide_scalar_value_t &val, halide_type_t &type) const noexcept { + void get_bound_const(int i, halide_scalar_value_t &val, Type &type) const noexcept { val = bound_const[i]; type = bound_const_type[i]; } + // Overload for the hot matching path: hand back the compact bound-const type + // directly so a re-match can compare types in 4-byte space without + // materializing (and re-comparing) two full 8-byte Types. HALIDE_ALWAYS_INLINE - // NOLINTNEXTLINE(modernize-use-equals-default): Can't use `= default`; clang-tidy complains about noexcept mismatch - MatcherState() noexcept { + void get_bound_const(int i, halide_scalar_value_t &val, BoundConstType &type) const noexcept { + val = bound_const[i]; + type = bound_const_type[i]; } + + HALIDE_ALWAYS_INLINE + MatcherState() noexcept = default; }; template::type::pattern_tag> + typename = typename std::remove_reference_t::pattern_tag> struct enable_if_pattern { struct type {}; }; template struct bindings { - constexpr static uint32_t mask = std::remove_reference::type::binds; + constexpr static uint32_t mask = std::remove_reference_t::binds; }; HALIDE_ALWAYS_INLINE -Expr make_const_expr(halide_scalar_value_t val, halide_type_t ty) { - halide_type_t scalar_type = ty; - const int lanes = scalar_type.lanes; - scalar_type.lanes = 1; +Expr make_const_expr(halide_scalar_value_t val, Type ty) { + Type scalar_type = ty; + const int lanes = scalar_type.lanes(); + scalar_type = scalar_type.with_lanes(1); Expr e; - switch (scalar_type.code) { + switch (scalar_type.code()) { case halide_type_int: e = IntImm::make(scalar_type, val.u.i64); break; @@ -189,7 +225,7 @@ struct SpecificExpr { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return Expr(&expr); } @@ -224,9 +260,9 @@ struct WildConstInt { int64_t value = ((const IntImm *)op)->value; if (bound & binds) { halide_scalar_value_t val; - halide_type_t type; + BoundConstType type; state.get_bound_const(i, val, type); - return (halide_type_t)e.type == type && value == val.u.i64; + return type == BoundConstType(e.type) && value == val.u.i64; } state.set_bound_const(i, value, e.type); return true; @@ -237,7 +273,7 @@ struct WildConstInt { static_assert(i >= 0 && i < max_wild, "Wild with out-of-range index"); if (bound & binds) { halide_scalar_value_t val; - halide_type_t type; + Type type; state.get_bound_const(i, val, type); return type == i64_type && value == val.u.i64; } @@ -246,16 +282,16 @@ struct WildConstInt { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t val; - halide_type_t type; + Type type; state.get_bound_const(i, val, type); return make_const_expr(val, type); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { state.get_bound_const(i, val, ty); return false; } @@ -290,25 +326,25 @@ struct WildConstUInt { uint64_t value = ((const UIntImm *)op)->value; if (bound & binds) { halide_scalar_value_t val; - halide_type_t type; + BoundConstType type; state.get_bound_const(i, val, type); - return (halide_type_t)e.type == type && value == val.u.u64; + return type == BoundConstType(e.type) && value == val.u.u64; } state.set_bound_const(i, value, e.type); return true; } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t val; - halide_type_t type; + Type type; state.get_bound_const(i, val, type); return make_const_expr(val, type); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { state.get_bound_const(i, val, ty); return false; } @@ -343,25 +379,25 @@ struct WildConstFloat { double value = ((const FloatImm *)op)->value; if (bound & binds) { halide_scalar_value_t val; - halide_type_t type; + BoundConstType type; state.get_bound_const(i, val, type); - return (halide_type_t)e.type == type && value == val.u.f64; + return type == BoundConstType(e.type) && value == val.u.f64; } state.set_bound_const(i, value, e.type); return true; } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t val; - halide_type_t type; + Type type; state.get_bound_const(i, val, type); return make_const_expr(val, type); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { state.get_bound_const(i, val, ty); return false; } @@ -410,16 +446,16 @@ struct WildConst { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t val; - halide_type_t type; + Type type; state.get_bound_const(i, val, type); return make_const_expr(val, type); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { state.get_bound_const(i, val, ty); return false; } @@ -452,7 +488,7 @@ struct Wild { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return state.get_binding(i); } @@ -516,15 +552,15 @@ struct IntLiteral { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return make_const(type_hint, v); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { // Assume type is already correct - switch (ty.code) { + switch (ty.code()) { case halide_type_int: val.u.i64 = v; break; @@ -591,13 +627,13 @@ inline std::ostream &operator<<(std::ostream &s, const IntLiteral &op) { } template -int64_t constant_fold_bin_op(halide_type_t &, int64_t, int64_t, bool &) noexcept; +int64_t constant_fold_bin_op(Type &, int64_t, int64_t, bool &) noexcept; template -uint64_t constant_fold_bin_op(halide_type_t &, uint64_t, uint64_t, bool &) noexcept; +uint64_t constant_fold_bin_op(Type &, uint64_t, uint64_t, bool &) noexcept; template -double constant_fold_bin_op(halide_type_t &, double, double, bool &) noexcept; +double constant_fold_bin_op(Type &, double, double, bool &) noexcept; constexpr bool commutative(IRNodeType t) { return (t == IRNodeType::Add || @@ -647,7 +683,7 @@ struct BinOp { constexpr static bool foldable = A::foldable && B::foldable; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = false; halide_scalar_value_t val_a, val_b; if (std::is_same_v) { @@ -669,7 +705,7 @@ struct BinOp { } overflow |= b.make_folded_const(val_b, ty, state); } - switch (ty.code) { + switch (ty.code()) { case halide_type_int: val.u.i64 = constant_fold_bin_op(ty, val_a.u.i64, val_b.u.i64, overflow); break; @@ -688,7 +724,7 @@ struct BinOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { Expr ea, eb; if (std::is_same_v) { eb = b.make(state, type_hint); @@ -746,7 +782,7 @@ struct CmpOp { constexpr static bool foldable = A::foldable && B::foldable; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = false; halide_scalar_value_t val_a, val_b; // If one side is an untyped const, evaluate the other side first to get a type hint. @@ -758,7 +794,7 @@ struct CmpOp { overflow |= b.make_folded_const(val_b, ty, state); } - switch (ty.code) { + switch (ty.code()) { case halide_type_int: val.u.u64 = constant_fold_cmp_op(val_a.u.i64, val_b.u.i64); break; @@ -773,13 +809,12 @@ struct CmpOp { // unreachable ; } - ty.code = halide_type_uint; - ty.bits = 1; + ty = Bool(ty.lanes()); return overflow; } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { // If one side is an untyped const, evaluate the other side first to get a type hint. Expr ea, eb; if (std::is_same_v) { @@ -898,21 +933,21 @@ HALIDE_ALWAYS_INLINE auto add(A &&a, B &&b) -> decltype(IRMatcher::operator+(a, } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { - overflow |= (t.bits >= 32) && add_would_overflow(t.bits, a, b); - int dead_bits = 64 - t.bits; +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { + overflow |= (t.bits() >= 32) && add_would_overflow(t.bits(), a, b); + int dead_bits = 64 - t.bits(); // Drop the high bits then sign-extend them back return int64_t((uint64_t(a) + uint64_t(b)) << dead_bits) >> dead_bits; } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { uint64_t ones = (uint64_t)(-1); - return (a + b) & (ones >> (64 - t.bits)); + return (a + b) & (ones >> (64 - t.bits())); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return a + b; } @@ -931,21 +966,21 @@ HALIDE_ALWAYS_INLINE auto sub(A &&a, B &&b) -> decltype(IRMatcher::operator-(a, } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { - overflow |= (t.bits >= 32) && sub_would_overflow(t.bits, a, b); +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { + overflow |= (t.bits() >= 32) && sub_would_overflow(t.bits(), a, b); // Drop the high bits then sign-extend them back - int dead_bits = 64 - t.bits; + int dead_bits = 64 - t.bits(); return int64_t((uint64_t(a) - uint64_t(b)) << dead_bits) >> dead_bits; } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { uint64_t ones = (uint64_t)(-1); - return (a - b) & (ones >> (64 - t.bits)); + return (a - b) & (ones >> (64 - t.bits())); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return a - b; } @@ -964,21 +999,21 @@ HALIDE_ALWAYS_INLINE auto mul(A &&a, B &&b) -> decltype(IRMatcher::operator*(a, } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { - overflow |= (t.bits >= 32) && mul_would_overflow(t.bits, a, b); - int dead_bits = 64 - t.bits; +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { + overflow |= (t.bits() >= 32) && mul_would_overflow(t.bits(), a, b); + int dead_bits = 64 - t.bits(); // Drop the high bits then sign-extend them back return int64_t((uint64_t(a) * uint64_t(b)) << dead_bits) >> dead_bits; } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { uint64_t ones = (uint64_t)(-1); - return (a * b) & (ones >> (64 - t.bits)); + return (a * b) & (ones >> (64 - t.bits())); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return a * b; } @@ -995,17 +1030,17 @@ HALIDE_ALWAYS_INLINE auto div(A &&a, B &&b) -> decltype(IRMatcher::operator/(a, } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op
(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op
(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return div_imp(a, b); } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op
(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op
(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return div_imp(a, b); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op
(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op
(Type &t, double a, double b, bool &overflow) noexcept { return div_imp(a, b); } @@ -1024,17 +1059,17 @@ HALIDE_ALWAYS_INLINE auto mod(A &&a, B &&b) -> decltype(IRMatcher::operator%(a, } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return mod_imp(a, b); } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return mod_imp(a, b); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return mod_imp(a, b); } @@ -1046,17 +1081,17 @@ HALIDE_ALWAYS_INLINE auto min(A &&a, B &&b) noexcept -> BinOp -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return std::min(a, b); } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return std::min(a, b); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return std::min(a, b); } @@ -1068,17 +1103,17 @@ HALIDE_ALWAYS_INLINE auto max(A &&a, B &&b) noexcept -> BinOp -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return std::max(a, b); } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return std::max(a, b); } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { return std::max(a, b); } @@ -1243,17 +1278,17 @@ HALIDE_ALWAYS_INLINE auto or_op(A &&a, B &&b) -> decltype(IRMatcher::operator||( } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return (a | b) & 1; } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return (a | b) & 1; } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { // Unreachable, as it would be a type mismatch. return 0; } @@ -1269,17 +1304,17 @@ HALIDE_ALWAYS_INLINE auto and_op(A &&a, B &&b) -> decltype(IRMatcher::operator&& } template<> -HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(halide_type_t &t, int64_t a, int64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE int64_t constant_fold_bin_op(Type &t, int64_t a, int64_t b, bool &overflow) noexcept { return a & b & 1; } template<> -HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(halide_type_t &t, uint64_t a, uint64_t b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE uint64_t constant_fold_bin_op(Type &t, uint64_t a, uint64_t b, bool &overflow) noexcept { return a & b & 1; } template<> -HALIDE_ALWAYS_INLINE double constant_fold_bin_op(halide_type_t &t, double a, double b, bool &overflow) noexcept { +HALIDE_ALWAYS_INLINE double constant_fold_bin_op(Type &t, double a, double b, bool &overflow) noexcept { // Unreachable return 0; } @@ -1311,7 +1346,7 @@ struct OptionalIntrinType { template<> struct OptionalIntrinType { - halide_type_t type; + Type type; bool check(const Type &t) const { return t == Type(type); } @@ -1378,7 +1413,7 @@ struct Intrin { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { Expr arg0 = std::get<0>(args).make(state, type_hint); if (intrin == Call::likely) { return likely(std::move(arg0)); @@ -1438,7 +1473,7 @@ struct Intrin { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = false; halide_scalar_value_t arg1; // Assuming the args have the same type as the intrinsic is incorrect in @@ -1446,15 +1481,14 @@ struct Intrin { // has the same type as the intrinsic, and we can always treat the RHS // as a signed int, because we're using 64 bits for it. overflow |= std::get<0>(args).make_folded_const(val, ty, state); - halide_type_t signed_ty = ty; - signed_ty.code = halide_type_int; + Type signed_ty = ty.with_code(halide_type_int); // We can just directly get the second arg here, because we only want to // instantiate this method for shifts, which have two args. overflow |= std::get<1>(args).make_folded_const(arg1, signed_ty, state); if (intrin == Call::shift_left) { if (arg1.u.i64 < 0) { - if (ty.code == halide_type_int) { + if (ty.code() == halide_type_int) { // Arithmetic shift val.u.i64 >>= -arg1.u.i64; } else { @@ -1466,7 +1500,7 @@ struct Intrin { } } else if (intrin == Call::shift_right) { if (arg1.u.i64 > 0) { - if (ty.code == halide_type_int) { + if (ty.code() == halide_type_int) { // Arithmetic shift val.u.i64 >>= arg1.u.i64; } else { @@ -1618,14 +1652,14 @@ struct NotOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return Not::make(a.make(state, type_hint)); } constexpr static bool foldable = A::foldable; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = a.make_folded_const(val, ty, state); val.u.u64 = ~val.u.u64; val.u.u64 &= 1; @@ -1743,17 +1777,17 @@ struct SelectOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return Select::make(c.make(state, {}), t.make(state, type_hint), f.make(state, type_hint)); } constexpr static bool foldable = C::foldable && T::foldable && F::foldable; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = false; - halide_scalar_value_t c_val, t_val, f_val; - halide_type_t c_ty; + halide_scalar_value_t c_val; + Type c_ty(halide_type_int, 0, 0); overflow |= c.make_folded_const(c_val, c_ty, state); if ((c_val.u.u64 & 1) == 1) { overflow |= t.make_folded_const(val, ty, state); @@ -1810,14 +1844,14 @@ struct BroadcastOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t lanes_val; - halide_type_t ty; + Type ty(halide_type_int, 0, 0); bool overflow = false; overflow |= lanes.make_folded_const(lanes_val, ty, state); internal_assert(!overflow) << "Overflow occurred computing the lanes field of a Broadcast node"; int32_t l = (int32_t)lanes_val.u.i64; - type_hint.lanes /= l; + type_hint = type_hint.with_lanes(type_hint.lanes() / l); Expr val = a.make(state, type_hint); if (l == 1) { return val; @@ -1829,14 +1863,14 @@ struct BroadcastOp { constexpr static bool foldable = false; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = false; halide_scalar_value_t lanes_val; - halide_type_t lanes_ty; + Type lanes_ty(halide_type_int, 0, 0); overflow |= lanes.make_folded_const(lanes_val, lanes_ty, state); uint16_t l = (uint16_t)lanes_val.u.i64; overflow |= a.make_folded_const(val, ty, state); - ty.lanes = l; + ty = ty.with_lanes(l); return overflow; } }; @@ -1890,15 +1924,15 @@ struct RampOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t lanes_val; - halide_type_t ty; + Type ty(halide_type_int, 0, 0); bool overflow = false; overflow |= lanes.make_folded_const(lanes_val, ty, state); internal_assert(!overflow) << "Overflow occurred computing the lanes field of a Ramp node in a rewriter rule."; int32_t l = (int32_t)lanes_val.u.i64; - type_hint.lanes /= l; + type_hint = type_hint.with_lanes(type_hint.lanes() / l); Expr ea, eb; eb = b.make(state, type_hint); ea = a.make(state, eb.type()); @@ -1955,9 +1989,9 @@ struct VectorReduceOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t lanes_val; - halide_type_t ty; + Type ty(halide_type_int, 0, 0); bool overflow = false; overflow |= lanes.make_folded_const(lanes_val, ty, state); internal_assert(!overflow) @@ -2046,7 +2080,7 @@ struct NegateOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { Expr ea = a.make(state, type_hint); Expr z = make_zero(ea.type()); return Sub::make(std::move(z), std::move(ea)); @@ -2055,12 +2089,12 @@ struct NegateOp { constexpr static bool foldable = A::foldable; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { bool overflow = a.make_folded_const(val, ty, state); - int dead_bits = 64 - ty.bits; - switch (ty.code) { + int dead_bits = 64 - ty.bits(); + switch (ty.code()) { case halide_type_int: - if (ty.bits >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits)) == 0) { + if (ty.bits() >= 32 && val.u.u64 && (val.u.u64 << (65 - ty.bits())) == 0) { // Trying to negate the most negative signed int for a no-overflow type. overflow = true; } else { @@ -2128,7 +2162,7 @@ struct CastOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return cast(t, a.make(state, {})); } @@ -2142,7 +2176,7 @@ std::ostream &operator<<(std::ostream &s, const CastOp &op) { } template -HALIDE_ALWAYS_INLINE auto cast(halide_type_t t, A &&a) noexcept -> CastOp { +HALIDE_ALWAYS_INLINE auto cast(Type t, A &&a) noexcept -> CastOp { assert_is_lvalue_if_expr(); return {t, pattern_arg(a)}; } @@ -2173,7 +2207,7 @@ struct WidenOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { Expr e = a.make(state, {}); Type w = e.type().widen(); return cast(w, std::move(e)); @@ -2223,9 +2257,9 @@ struct SliceOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t base_val, stride_val, lanes_val; - halide_type_t ty; + Type ty(halide_type_int, 0, 0); bool overflow = false; overflow |= base.make_folded_const(base_val, ty, state); int b = (int)base_val.u.i64; @@ -2287,9 +2321,9 @@ struct TransposeOp { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { halide_scalar_value_t factor_val; - halide_type_t ty; + Type ty(halide_type_int, 0, 0); bool overflow = false; overflow |= factor.make_folded_const(factor_val, ty, state); internal_assert(!overflow) @@ -2331,9 +2365,9 @@ struct Fold { constexpr static bool canonical = true; HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const noexcept { + Expr make(MatcherState &state, Type type_hint) const noexcept { halide_scalar_value_t c; - halide_type_t ty = type_hint; + Type ty = type_hint; bool overflow = false; overflow |= a.make_folded_const(c, ty, state); if (overflow) { @@ -2344,14 +2378,13 @@ struct Fold { // (e.g. because it's from an int literal). Make the type code // and bits match the required type, if there is one (we can // tell from the bits field). - if (type_hint.bits) { - if (((int)ty.code == (int)halide_type_int) && - ((int)type_hint.code == (int)halide_type_float)) { + if (type_hint.bits()) { + if (((int)ty.code() == (int)halide_type_int) && + ((int)type_hint.code() == (int)halide_type_float)) { int64_t x = c.u.i64; c.u.f64 = (double)x; } - ty.code = type_hint.code; - ty.bits = type_hint.bits; + ty = type_hint.with_lanes(ty.lanes()); } return make_const_expr(c, ty); @@ -2360,7 +2393,7 @@ struct Fold { constexpr static bool foldable = A::foldable; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { return a.make_folded_const(val, ty, state); } }; @@ -2393,15 +2426,13 @@ struct Overflows { constexpr static bool foldable = A::foldable; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { // Run the inner fold and consume its overflow flag (the whole // point of this predicate is to ask about it without letting it // taint the surrounding evaluation). const bool inner_overflow = a.make_folded_const(val, ty, state); val.u.u64 = inner_overflow ? 1 : 0; - ty.code = halide_type_uint; - ty.bits = 64; - ty.lanes = 1; + ty = UInt(64, 1); return false; } }; @@ -2438,13 +2469,13 @@ struct Overflow { } HALIDE_ALWAYS_INLINE - Expr make(MatcherState &state, halide_type_t type_hint) const { + Expr make(MatcherState &state, Type type_hint) const { return make_signed_integer_overflow(type_hint); } constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { val.u.u64 = 0; return true; } @@ -2473,11 +2504,9 @@ struct IsConst { constexpr static bool foldable = true; template - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const noexcept { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const noexcept { Expr e = a.make(state, {}); - ty.code = halide_type_uint; - ty.bits = 64; - ty.lanes = 1; + ty = UInt(64, 1); if (check_v) { val.u.u64 = ::Halide::Internal::is_const(e, v) ? 1 : 0; } else { @@ -2525,13 +2554,11 @@ struct CanProve { constexpr static bool foldable = true; // Includes a raw call to an inlined make method, so don't inline. - [[nodiscard]] HALIDE_NEVER_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_NEVER_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { Expr condition = a.make(state, {}); condition = prover->mutate(condition, nullptr); val.u.u64 = is_const_one(condition); - ty.code = halide_type_uint; - ty.bits = 1; - ty.lanes = condition.type().lanes(); + ty = Bool(condition.type().lanes()); return false; } }; @@ -2562,13 +2589,11 @@ struct IsFloat { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); val.u.u64 = t.is_float(); - ty.code = halide_type_uint; - ty.bits = 1; - ty.lanes = t.lanes(); + ty = Bool(t.lanes()); return false; } }; @@ -2601,13 +2626,11 @@ struct IsInt { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); val.u.u64 = t.is_int() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes); - ty.code = halide_type_uint; - ty.bits = 1; - ty.lanes = t.lanes(); + ty = Bool(t.lanes()); return false; } }; @@ -2647,13 +2670,11 @@ struct IsUInt { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); val.u.u64 = t.is_uint() && (bits == 0 || t.bits() == bits) && (lanes == 0 || t.lanes() == lanes); - ty.code = halide_type_uint; - ty.bits = 1; - ty.lanes = t.lanes(); + ty = Bool(t.lanes()); return false; } }; @@ -2691,13 +2712,11 @@ struct IsScalar { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); val.u.u64 = t.is_scalar(); - ty.code = halide_type_uint; - ty.bits = 1; - ty.lanes = t.lanes(); + ty = Bool(t.lanes()); return false; } }; @@ -2728,17 +2747,16 @@ struct IsMaxValue { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. const bool overflow = a.make_folded_const(val, ty, state); - const uint64_t max_bits = (uint64_t)(-1) >> (64 - ty.bits + (ty.code == halide_type_int)); - if (ty.code == halide_type_uint || ty.code == halide_type_int) { + const uint64_t max_bits = (uint64_t)(-1) >> (64 - ty.bits() + (ty.code() == halide_type_int)); + if (ty.code() == halide_type_uint || ty.code() == halide_type_int) { val.u.u64 = (val.u.u64 == max_bits); } else { val.u.u64 = 0; } - ty.code = halide_type_uint; - ty.bits = 1; + ty = Bool(ty.lanes()); return overflow; } }; @@ -2769,19 +2787,18 @@ struct IsMinValue { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. const bool overflow = a.make_folded_const(val, ty, state); - if (ty.code == halide_type_int) { - const uint64_t min_bits = (uint64_t)(-1) << (ty.bits - 1); + if (ty.code() == halide_type_int) { + const uint64_t min_bits = (uint64_t)(-1) << (ty.bits() - 1); val.u.u64 = (val.u.u64 == min_bits); - } else if (ty.code == halide_type_uint) { + } else if (ty.code() == halide_type_uint) { val.u.u64 = (val.u.u64 == 0); } else { val.u.u64 = 0; } - ty.code = halide_type_uint; - ty.bits = 1; + ty = Bool(ty.lanes()); return overflow; } }; @@ -2812,13 +2829,11 @@ struct LanesOf { constexpr static bool foldable = true; - [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, halide_type_t &ty, MatcherState &state) const { + [[nodiscard]] HALIDE_ALWAYS_INLINE bool make_folded_const(halide_scalar_value_t &val, Type &ty, MatcherState &state) const { // a is almost certainly a very simple pattern (e.g. a wild), so just inline the make method. Type t = a.make(state, {}).type(); val.u.u64 = t.lanes(); - ty.code = halide_type_uint; - ty.bits = 32; - ty.lanes = 1; + ty = UInt(32); return false; } }; @@ -2842,20 +2857,21 @@ template::foldable && std::decay_t::foldable>> HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, - halide_type_t wildcard_type, halide_type_t output_type) noexcept { + Type wildcard_type, Type output_type) noexcept { // We only validate the rules in the scalar case - wildcard_type.lanes = output_type.lanes = 1; + wildcard_type = wildcard_type.with_lanes(1); + output_type = output_type.with_lanes(1); // Track which types this rule has been tested for before - static std::set tested; + static std::set tested; - if (!tested.insert(reinterpret_bits(wildcard_type)).second) { + if (!tested.insert(wildcard_type).second) { return; } // Print it in a form where it can be piped into a python/z3 validator - debug(0) << "validate('" << before << "', '" << after << "', '" << pred << "', " << Type(wildcard_type) << ", " << Type(output_type) << ")\n"; + debug(0) << "validate('" << before << "', '" << after << "', '" << pred << "', " << wildcard_type << ", " << output_type << ")\n"; // Substitute some random constants into the before and after // expressions and see if the rule holds true. This should catch @@ -2870,11 +2886,11 @@ HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicat // We want to test small constants more frequently than // large ones, otherwise we'll just get coverage of // overflow rules. - int shift = (int)(rng() & (wildcard_type.bits - 1)); + int shift = (int)(rng() & (wildcard_type.bits() - 1)); for (int i = 0; i < max_wild; i++) { // Bind all the exprs and constants - switch (wildcard_type.code) { + switch (wildcard_type.code()) { case halide_type_uint: { // Normalize to the type's range by adding zero uint64_t val = constant_fold_bin_op(wildcard_type, (uint64_t)rng() >> shift, (uint64_t)0, overflow); @@ -2905,8 +2921,8 @@ HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicat state.set_binding(i, *exprs[i].get()); } - halide_scalar_value_t val_pred, val_before, val_after; - halide_type_t type = output_type; + halide_scalar_value_t val_before, val_after; + Type type = output_type; if (!evaluate_predicate(pred, state)) { continue; } @@ -2917,7 +2933,7 @@ HALIDE_NEVER_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicat } bool ok = true; - switch (output_type.code) { + switch (output_type.code()) { case halide_type_uint: // Compare normalized representations ok &= (constant_fold_bin_op(output_type, val_before.u.u64, (uint64_t)0, overflow) == @@ -2965,7 +2981,7 @@ template::foldable && std::decay_t::foldable)>> HALIDE_ALWAYS_INLINE void fuzz_test_rule(Before &&before, After &&after, Predicate &&pred, - halide_type_t, halide_type_t, int dummy = 0) noexcept { + Type, Type, int dummy = 0) noexcept { // We can't verify rewrite rules that can't be constant-folded. } @@ -2978,7 +2994,7 @@ template::type> HALIDE_ALWAYS_INLINE bool evaluate_predicate(Pattern p, MatcherState &state) { halide_scalar_value_t c; - halide_type_t ty = halide_type_of(); + Type ty = Bool(); const bool overflow = p.make_folded_const(c, ty, state); // Overflow counts as a failed predicate. return (c.u.u64 != 0) && !overflow; @@ -3001,11 +3017,11 @@ struct Rewriter { Instance instance; Expr result; MatcherState state; - halide_type_t output_type, wildcard_type; + Type output_type, wildcard_type; bool validate; HALIDE_ALWAYS_INLINE - Rewriter(Instance instance, halide_type_t output, halide_type_t wildcard) + Rewriter(Instance instance, Type output, Type wildcard) : instance(std::move(instance)), output_type(output), wildcard_type(wildcard) { } @@ -3180,18 +3196,18 @@ struct Rewriter { // @{ template::type> -HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type, halide_type_t wildcard_type) noexcept -> Rewriter { +HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, Type output_type, Type wildcard_type) noexcept -> Rewriter { return {pattern_arg(instance), output_type, wildcard_type}; } template::type> -HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, halide_type_t output_type) noexcept -> Rewriter { +HALIDE_ALWAYS_INLINE auto rewriter(Instance instance, Type output_type) noexcept -> Rewriter { return {pattern_arg(instance), output_type, output_type}; } HALIDE_ALWAYS_INLINE -auto rewriter(const Expr &e, halide_type_t wildcard_type) noexcept -> Rewriter { +auto rewriter(const Expr &e, Type wildcard_type) noexcept -> Rewriter { return {pattern_arg(e), e.type(), wildcard_type}; } diff --git a/src/OffloadGPULoops.cpp b/src/OffloadGPULoops.cpp index 7351a038e019..c040dd72e39b 100644 --- a/src/OffloadGPULoops.cpp +++ b/src/OffloadGPULoops.cpp @@ -196,7 +196,7 @@ class InjectGpuOffload : public IRMutator { args.emplace_back(val); if (runtime_run_takes_types) { - arg_types_or_sizes.emplace_back(((halide_type_t)i.type).as_u32()); + arg_types_or_sizes.emplace_back(i.type.to_abi()); } else { arg_types_or_sizes.emplace_back(cast(target_size_t_type, i.is_buffer ? 8 : i.type.bytes())); } diff --git a/src/Param.h b/src/Param.h index 30864c2db435..a81106110019 100644 --- a/src/Param.h +++ b/src/Param.h @@ -185,14 +185,14 @@ class Param { // it just to reduce code size for the common case of T != void. #define HALIDE_HANDLE_TYPE_DISPATCH(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): \ + case halide_type_t(CODE, BITS): \ user_assert(Internal::IsRoundtrippable::value(val)) \ << "The value " << val << " cannot be losslessly converted to type " << type; \ param.set_scalar(Internal::StaticCast::value(val)); \ break; - const Type type = param.type(); - switch (((halide_type_t)type).element_of().as_u32()) { + const halide_type_t type = param.type().to_abi(); + switch (type) { HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 32, float) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 64, double) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_int, 8, int8_t) @@ -262,14 +262,14 @@ class Param { // it just to reduce code size for the common case of T != void. #define HALIDE_HANDLE_TYPE_DISPATCH(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): \ + case halide_type_t(CODE, BITS): \ user_assert(Internal::IsRoundtrippable::value(val)) \ << "The value " << val << " cannot be losslessly converted to type " << type; \ param.set_estimate(Expr(Internal::StaticCast::value(val))); \ break; - const Type type = param.type(); - switch (((halide_type_t)type).element_of().as_u32()) { + const halide_type_t type = param.type().to_abi(); + switch (type) { HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 32, float) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_float, 64, double) HALIDE_HANDLE_TYPE_DISPATCH(halide_type_int, 8, int8_t) diff --git a/src/Pipeline.cpp b/src/Pipeline.cpp index f4dca6ee3292..b5c8ac5ad695 100644 --- a/src/Pipeline.cpp +++ b/src/Pipeline.cpp @@ -1126,8 +1126,8 @@ void Pipeline::infer_input_bounds(JITUserContext *context, internal_assert(ia.param.defined() && ia.param.is_buffer()); // Make some empty Buffers of the right dimensionality vector initial_shape(ia.param.dimensions(), 0); - tracked_buffers[i].query = Runtime::Buffer<>(ia.param.type(), nullptr, initial_shape); - tracked_buffers[i].orig = Runtime::Buffer<>(ia.param.type(), nullptr, initial_shape); + tracked_buffers[i].query = Runtime::Buffer<>(ia.param.type().to_abi(), nullptr, initial_shape); + tracked_buffers[i].orig = Runtime::Buffer<>(ia.param.type().to_abi(), nullptr, initial_shape); args.store[i] = tracked_buffers[i].query.raw_buffer(); } } diff --git a/src/PythonExtensionGen.cpp b/src/PythonExtensionGen.cpp index deb6b45f70f5..50b5f3b0ba12 100644 --- a/src/PythonExtensionGen.cpp +++ b/src/PythonExtensionGen.cpp @@ -239,7 +239,6 @@ bool unpack_buffer(PyObject *py_obj, return false; } } - halide_buf.type.lanes = 1; halide_buf.dimensions = py_buf.ndim; halide_buf.dim = halide_dim; halide_buf.host = (uint8_t *)py_buf.buf; diff --git a/src/Type.cpp b/src/Type.cpp index cbe07a5b509b..824e105a40e4 100644 --- a/src/Type.cpp +++ b/src/Type.cpp @@ -1,6 +1,7 @@ #include "ConstantBounds.h" #include "IR.h" #include +#include #include namespace Halide { @@ -222,9 +223,54 @@ bool Type::can_represent(double x) const { } } +namespace Internal { +namespace { +std::mutex &handle_type_intern_mutex() { + static std::mutex m; + return m; +} +std::vector &handle_type_intern_table() { + static std::vector t; + return t; +} +} // namespace + +uint32_t intern_handle_type(const halide_handle_cplusplus_type *handle_type) { + // The overwhelmingly common case: a non-handle type. Costs a null check, + // no lock. + if (handle_type == nullptr) { + return 0; + } + std::scoped_lock lock(handle_type_intern_mutex()); + auto &table = handle_type_intern_table(); + // Dedup by pointer identity. Handle types are rare and few, so a linear + // scan is cheap; the pointees are externally owned and stable, so pointer + // identity is a safe key. (Distinct pointers with equal content still + // compare equal via Type::same_handle_type's deep comparison.) + for (size_t i = 0; i < table.size(); i++) { + if (table[i] == handle_type) { + return (uint32_t)(i + 1); // +1: index 0 is reserved for null + } + } + table.push_back(handle_type); + internal_assert(table.size() < (size_t)0xffffffffu) << "handle-type intern table overflow"; + return (uint32_t)table.size(); +} + +const halide_handle_cplusplus_type *get_interned_handle_type(uint32_t index) { + if (index == 0) { + return nullptr; + } + std::scoped_lock lock(handle_type_intern_mutex()); + auto &table = handle_type_intern_table(); + internal_assert(index <= table.size()) << "invalid handle-type intern index"; + return table[index - 1]; +} +} // namespace Internal + bool Type::same_handle_type(const Type &other) const { - const halide_handle_cplusplus_type *first = handle_type; - const halide_handle_cplusplus_type *second = other.handle_type; + const halide_handle_cplusplus_type *first = handle_type(); + const halide_handle_cplusplus_type *second = other.handle_type(); if (first == second) { return true; @@ -264,37 +310,40 @@ std::string type_to_c_type(Type type, bool include_space, bool c_plus_plus) { } else if (type.is_handle()) { needs_space = false; + // Resolve the interned handle metadata once rather than re-lookup per use. + const halide_handle_cplusplus_type *ht = type.handle_type(); + // If there is no type info or is generating C (not C++) and // the type is a class or in an inner scope, just use void *. - if (type.handle_type == nullptr || + if (ht == nullptr || (!c_plus_plus && - (!type.handle_type->namespaces.empty() || - !type.handle_type->enclosing_types.empty() || - type.handle_type->inner_name.cpp_type_type == halide_cplusplus_type_name::Class))) { + (!ht->namespaces.empty() || + !ht->enclosing_types.empty() || + ht->inner_name.cpp_type_type == halide_cplusplus_type_name::Class))) { oss << "void *"; } else { - if (type.handle_type->inner_name.cpp_type_type == + if (ht->inner_name.cpp_type_type == halide_cplusplus_type_name::Struct) { oss << "struct "; } - if (!type.handle_type->namespaces.empty() || - !type.handle_type->enclosing_types.empty()) { + if (!ht->namespaces.empty() || + !ht->enclosing_types.empty()) { oss << "::"; - for (const auto &ns : type.handle_type->namespaces) { + for (const auto &ns : ht->namespaces) { oss << ns << "::"; } - for (const auto &enclosing_type : type.handle_type->enclosing_types) { + for (const auto &enclosing_type : ht->enclosing_types) { oss << enclosing_type.name << "::"; } } - oss << type.handle_type->inner_name.name; - if (type.handle_type->reference_type == halide_handle_cplusplus_type::LValueReference) { + oss << ht->inner_name.name; + if (ht->reference_type == halide_handle_cplusplus_type::LValueReference) { oss << " &"; - } else if (type.handle_type->reference_type == halide_handle_cplusplus_type::RValueReference) { + } else if (ht->reference_type == halide_handle_cplusplus_type::RValueReference) { oss << " &&"; } - for (auto modifier : type.handle_type->cpp_type_modifiers) { + for (auto modifier : ht->cpp_type_modifiers) { if (modifier & halide_handle_cplusplus_type::Const) { oss << " const"; } diff --git a/src/Type.h b/src/Type.h index 174c53de4961..a9622ecc77d9 100644 --- a/src/Type.h +++ b/src/Type.h @@ -6,6 +6,7 @@ #include "Util.h" #include "runtime/HalideRuntime.h" #include +#include /** \file * Defines halide types @@ -269,7 +270,17 @@ namespace Halide { namespace Internal { struct ConstantInterval; -} + +/** Handle (C++ pointer) types carry a pointer to externally-owned metadata + * describing the pointee type. To keep `Type` small (8 bytes) that metadata is + * referenced by a 4-byte index into a process-wide intern table rather than by + * an 8-byte pointer. Index 0 means "no handle metadata" (i.e. void *). The + * table stores the (non-owning) pointers callers supply — the pointees must + * outlive all `Type`s, exactly as before. `intern_handle_type(nullptr)` returns + * 0 without locking, so non-handle type construction pays nothing. */ +uint32_t intern_handle_type(const halide_handle_cplusplus_type *handle_type); +const halide_handle_cplusplus_type *get_interned_handle_type(uint32_t index); +} // namespace Internal struct Expr; @@ -280,7 +291,12 @@ struct Expr; * types. Instead vectorize a function. */ struct Type { private: - halide_type_t type; + halide_type_code_t type_code; + uint8_t type_bits; + uint16_t type_lanes; + // Index into the process-wide handle-type intern table (0 == no handle + // metadata). + uint32_t handle_index_ = 0; public: /** Aliases for halide_type_code_t values for legacy compatibility @@ -293,26 +309,35 @@ struct Type { static constexpr halide_type_code_t Handle = halide_type_handle; // @} + /** Exposed so code that needs to reason about the maximum representable + * lanes count (e.g. overflow checks when combining vectors) can derive + * it from Type itself instead of hardcoding a width. */ + static constexpr int kLanesBits = 8 * sizeof(type_lanes); + /** The number of bytes required to store a single scalar value of this type. Ignores vector lanes. */ int bytes() const { return (bits() + 7) / 8; } // Default ctor initializes everything to predictable-but-unlikely values - Type() - : type(Handle, 0, 0) { + constexpr Type() + : type_code(Handle), type_bits(0), type_lanes(0) { } /** Construct a runtime representation of a Halide type from: * code: The fundamental type from an enum. * bits: The bit size of one element. * lanes: The number of vector elements in the type. */ + HALIDE_ALWAYS_INLINE Type(halide_type_code_t code, int bits, int lanes, const halide_handle_cplusplus_type *handle_type = nullptr) - : type(code, (uint8_t)bits, (uint16_t)lanes), handle_type(handle_type) { - user_assert(lanes == type.lanes) + : type_code(code), type_bits((uint8_t)bits), type_lanes((uint16_t)lanes), + handle_index_(handle_type ? Internal::intern_handle_type(handle_type) : 0) { + internal_assert(lanes == (int)type_lanes) << "Halide only supports vector types with up to 65535 lanes. " << lanes << " lanes requested."; - user_assert(bits == type.bits) + internal_assert(bits == (int)type_bits) << "Halide only supports types with up to 255 bits. " << bits << " bits requested."; + internal_assert(code == Handle || !handle_type) + << "Cannot construct a non-handle Type with handle metadata.\n"; } /** Trivial copy constructor. */ @@ -321,58 +346,72 @@ struct Type { /** Trivial copy assignment operator. */ Type &operator=(const Type &that) = default; - /** Type is a wrapper around halide_type_t with more methods for use - * inside the compiler. This simply constructs the wrapper around - * the runtime value. */ + /** Construct a (scalar) language Type from an ABI element type. */ HALIDE_ALWAYS_INLINE - Type(const halide_type_t &that, const halide_handle_cplusplus_type *handle_type = nullptr) - : type(that), handle_type(handle_type) { + Type(halide_type_t that, const halide_handle_cplusplus_type *handle_type = nullptr) + : Type(that.code, that.bits, 1, handle_type) { } - /** Unwrap the runtime halide_type_t for use in runtime calls, etc. - * Representation is exactly equivalent. */ + /** Erase this language Type to the ABI/runtime halide_type_t for use in + * runtime calls, buffer/argument metadata, etc. This is a *checked* + * erasure: the ABI element type has no lanes, so a vector type + * (lanes >= 2) has no image here and asserts rather than silently + * dropping its lanes. */ HALIDE_ALWAYS_INLINE - operator halide_type_t() const { - return type; + halide_type_t to_abi() const { + internal_assert(type_lanes < 2) + << "Cannot erase a vector type with " << type_lanes + << " lanes to a scalar ABI halide_type_t.\n"; + return halide_type_t(code(), type_bits); } /** Return the underlying data type of an element as an enum value. */ HALIDE_ALWAYS_INLINE halide_type_code_t code() const { - return (halide_type_code_t)type.code; + return type_code; } /** Return the bit size of a single element of this type. */ HALIDE_ALWAYS_INLINE int bits() const { - return type.bits; + return type_bits; } /** Return the number of vector elements in this type. */ HALIDE_ALWAYS_INLINE int lanes() const { - return type.lanes; + return type_lanes; } /** Return Type with same number of bits and lanes, but new_code for a type code. */ + HALIDE_ALWAYS_INLINE Type with_code(halide_type_code_t new_code) const { - return Type(new_code, bits(), lanes(), - (new_code == code()) ? handle_type : nullptr); + Type t = *this; + t.type_code = new_code; + t.handle_index_ = new_code != code() ? 0 : handle_index_; // Changing the type code invalidates any handle metadata. + return t; } /** Return Type with same type code and lanes, but new_bits for the number of bits. */ + HALIDE_ALWAYS_INLINE Type with_bits(int new_bits) const { - return Type(code(), new_bits, lanes(), - (new_bits == bits()) ? handle_type : nullptr); + Type t = *this; + t.type_bits = (uint8_t)new_bits; + t.handle_index_ = new_bits != bits() ? 0 : handle_index_; // Changing the bit width invalidates any handle metadata. + return t; } /** Return Type with same type code and number of bits, * but new_lanes for the number of vector lanes. */ + HALIDE_ALWAYS_INLINE Type with_lanes(int new_lanes) const { - return Type(code(), bits(), new_lanes, handle_type); + Type t = *this; + t.type_lanes = (uint16_t)new_lanes; + return t; } /** Return Type with the same type code and number of lanes, but with at least twice as many bits. */ + HALIDE_ALWAYS_INLINE Type widen() const { if (is_bfloat()) { // Widening a bfloat16 should produce a float32. @@ -387,6 +426,7 @@ struct Type { } /** Return Type with the same type code and number of lanes, but with at most half as many bits. */ + HALIDE_ALWAYS_INLINE Type narrow() const { internal_assert(bits() != 1) << "Attempting to narrow a 1-bit type\n"; if (bits() == 8) { @@ -397,8 +437,15 @@ struct Type { } } - /** Type to be printed when declaring handles of this type. */ - const halide_handle_cplusplus_type *handle_type = nullptr; + /** The externally-owned C++ type metadata for a handle type (null for a + * plain void * handle or a non-handle type). Backed by a 4-byte intern + * index rather than stored inline, so this is now an accessor rather than a + * public field. */ + const halide_handle_cplusplus_type *handle_type() const { + // Inline the common (non-handle) case so it needs no call or table + // lookup; only a real handle index reaches the intern table. + return handle_index_ == 0 ? nullptr : Internal::get_interned_handle_type(handle_index_); + } /** Is this type boolean (represented as UInt(1))? */ HALIDE_ALWAYS_INLINE @@ -472,37 +519,42 @@ struct Type { bool same_handle_type(const Type &other) const; /** Compare two types for equality */ + HALIDE_ALWAYS_INLINE bool operator==(const Type &other) const { - return type == other.type && (code() != Handle || same_handle_type(other)); + return type_code == other.type_code && type_bits == other.type_bits && + type_lanes == other.type_lanes && (code() != Handle || same_handle_type(other)); } /** Compare two types for inequality */ bool operator!=(const Type &other) const { - return type != other.type || (code() == Handle && !same_handle_type(other)); + return !(*this == other); } - /** Compare two types for equality */ + /** Compare a language type to an ABI element type. Equal iff this type is a + * single element (not a vector) with the same code and bits. */ bool operator==(const halide_type_t &other) const { - return type == other; + return type_lanes < 2 && (uint8_t)type_code == (uint8_t)other.code && type_bits == other.bits; } /** Compare two types for inequality */ bool operator!=(const halide_type_t &other) const { - return type != other; + return !(*this == other); } /** Compare ordering of two types so they can be used in certain containers and algorithms */ bool operator<(const Type &other) const { - if (type < other.type) { + if (std::tie(type_code, type_bits, type_lanes) < + std::tie(other.type_code, other.type_bits, other.type_lanes)) { return true; } if (code() == Handle) { - return handle_type < other.handle_type; + return handle_type() < other.handle_type(); } return false; } /** Produce the scalar type (that of a single element) of this vector type */ + HALIDE_ALWAYS_INLINE Type element_of() const { return with_lanes(1); } @@ -539,33 +591,36 @@ struct Type { Expr min() const; }; +static_assert(sizeof(Type) == 8, "Halide::Type is a code+bits+lanes triple plus a 4-byte handle-type index"); +static_assert(std::is_trivially_copyable_v, "Type must stay trivially copyable"); + /** Constructing a signed integer type */ -inline Type Int(int bits, int lanes = 1) { +HALIDE_ALWAYS_INLINE Type Int(int bits, int lanes = 1) { return Type(Type::Int, bits, lanes); } /** Constructing an unsigned integer type */ -inline Type UInt(int bits, int lanes = 1) { +HALIDE_ALWAYS_INLINE Type UInt(int bits, int lanes = 1) { return Type(Type::UInt, bits, lanes); } /** Construct a floating-point type */ -inline Type Float(int bits, int lanes = 1) { +HALIDE_ALWAYS_INLINE Type Float(int bits, int lanes = 1) { return Type(Type::Float, bits, lanes); } /** Construct a floating-point type in the bfloat format. Only 16-bit currently supported. */ -inline Type BFloat(int bits, int lanes = 1) { +HALIDE_ALWAYS_INLINE Type BFloat(int bits, int lanes = 1) { return Type(Type::BFloat, bits, lanes); } /** Construct a boolean type */ -inline Type Bool(int lanes = 1) { +HALIDE_ALWAYS_INLINE Type Bool(int lanes = 1) { return UInt(1, lanes); } /** Construct a handle type */ -inline Type Handle(int lanes = 1, const halide_handle_cplusplus_type *handle_type = nullptr) { +HALIDE_ALWAYS_INLINE Type Handle(int lanes = 1, const halide_handle_cplusplus_type *handle_type = nullptr) { return Type(Type::Handle, 64, lanes, handle_type); } diff --git a/src/WasmExecutor.cpp b/src/WasmExecutor.cpp index 53e744d76468..0e7a24080446 100644 --- a/src/WasmExecutor.cpp +++ b/src/WasmExecutor.cpp @@ -388,11 +388,11 @@ std::vector compile_to_wasm(const Module &module, const std::string &fn_na template class Functor, typename... Args> auto dynamic_type_dispatch(const halide_type_t &type, Args &&...args) -> decltype(std::declval>()(std::forward(args)...)) { -#define HANDLE_CASE(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): \ +#define HANDLE_CASE(CODE, BITS, TYPE) \ + case halide_type_t(CODE, BITS): \ return Functor()(std::forward(args)...); - switch (type.element_of().as_u32()) { + switch (type) { HANDLE_CASE(halide_type_bfloat, 16, bfloat16_t) HANDLE_CASE(halide_type_float, 16, float16_t) HANDLE_CASE(halide_type_float, 32, float) @@ -458,7 +458,7 @@ bool build_extern_arg_types(const std::string &fn_name, const bool is_buffer = false; const bool is_ucon = false; // Specifying a type here with bits == 0 should trigger a proper 'void' return type - arg_types.emplace_back(ExternArgType{{halide_type_int, 0, 0}, is_void, is_buffer, is_ucon}); + arg_types.emplace_back(ExternArgType{halide_type_t{halide_type_int, 0}, is_void, is_buffer, is_ucon}); } else { const Type &t = sig.ret_type(); const bool is_void = false; @@ -472,7 +472,7 @@ bool build_extern_arg_types(const std::string &fn_name, user_assert(!(t.is_handle() && !is_buffer)) << "Halide Extern functions cannot return arbitrary pointers as arguments."; user_assert(!(t.is_int_or_uint() && t.bits() == 64)) << "Halide Extern functions cannot accept 64-bit values as arguments."; - arg_types.emplace_back(ExternArgType{t, is_void, is_buffer, is_ucon}); + arg_types.emplace_back(ExternArgType{t.to_abi(), is_void, is_buffer, is_ucon}); } for (size_t i = 0; i < arg_count; ++i) { @@ -489,7 +489,7 @@ bool build_extern_arg_types(const std::string &fn_name, user_assert(!(t.is_handle() && !is_buffer)) << "Halide Extern functions cannot accept arbitrary pointers as arguments."; user_assert(!(t.is_int_or_uint() && t.bits() == 64)) << "Halide Extern functions cannot accept 64-bit values as arguments."; - arg_types.emplace_back(ExternArgType{t, is_void, is_buffer, is_ucon}); + arg_types.emplace_back(ExternArgType{t.to_abi(), is_void, is_buffer, is_ucon}); } arg_types_out = std::move(arg_types); @@ -596,7 +596,7 @@ void dump_hostbuf(WabtContext &wabt_context, const halide_buffer_t *buf, const s } debug(1) << " }\n"; debug(1) << " flags = " << buf->flags << "\n"; - debug(1) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "," << buf->type.lanes << "\n"; + debug(1) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "\n"; debug(1) << " dimensions = " << buf->dimensions << "\n"; debug(1) << " dim = " << (void *)buf->dim << " = {\n"; for (int i = 0; i < buf->dimensions; i++) { @@ -627,7 +627,7 @@ void dump_wasmbuf(WabtContext &wabt_context, wasm32_ptr_t buf_ptr, const std::st } debug(1) << " }\n"; debug(1) << " flags = " << buf->flags << "\n"; - debug(1) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "," << buf->type.lanes << "\n"; + debug(1) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "\n"; debug(1) << " dimensions = " << buf->dimensions << "\n"; debug(1) << " dim = " << buf->dim << " -> " << (void *)dim << " = {\n"; for (int i = 0; i < buf->dimensions; i++) { @@ -836,7 +836,7 @@ inline wabt::interp::Value LoadValue::operator()(const void *src) { return wabt::interp::Value::Make(val); } -inline wabt::interp::Value load_value(const Type &t, const void *src) { +inline wabt::interp::Value load_value(const halide_type_t &t, const void *src) { return dynamic_type_dispatch(t, src); } @@ -876,7 +876,7 @@ inline void StoreValue::operator()(const wabt::interp::Value &src, v *(uint16_t *)dst = src.Get(); } -inline void store_value(const Type &t, const wabt::interp::Value &src, void *dst) { +inline void store_value(const halide_type_t &t, const wabt::interp::Value &src, void *dst) { dynamic_type_dispatch(t, src, dst); } @@ -1056,9 +1056,8 @@ WABT_HOST_CALLBACK(halide_trace_helper) { event.value = value_ptr ? ((void *)(base + value_ptr)) : nullptr; event.coordinates = coordinates_ptr ? ((int32_t *)(base + coordinates_ptr)) : nullptr; event.trace_tag = (const char *)(base + trace_tag_ptr); - event.type.code = (halide_type_code_t)type_code; - event.type.bits = (uint8_t)type_bits; - event.type.lanes = (uint16_t)type_lanes; + event.type = {(halide_type_code_t)type_code, (uint8_t)type_bits}; + event.lanes = type_lanes; event.event = (halide_trace_event_code_t)trace_code; event.parent_id = parent_id; event.thread_id = thread_id; @@ -1208,7 +1207,7 @@ wabt::Result extern_callback_wrapper(const std::vector &arg_types // int64 from a Value that is int32 will assert-fail. In JIT mode the value // doesn't even matter (except for guarding that it is our predicted constant). internal_assert(args[i].Get() == 0 || args[i].Get() == kMagicJitUserContextValue); - store_value(Int(32), args[i], &scalars[i]); + store_value(halide_type_of(), args[i], &scalars[i]); trampoline_args[i] = &scalars[i]; } else if (a.is_buffer) { const wasm32_ptr_t buf_ptr = args[i].Get(); @@ -1523,7 +1522,7 @@ void dump_hostbuf(const Local &context, const halide_buffer_t *buf, con } debug(0) << " }\n"; debug(0) << " flags = " << buf->flags << "\n"; - debug(0) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "," << buf->type.lanes << "\n"; + debug(0) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "\n"; debug(0) << " dimensions = " << buf->dimensions << "\n"; debug(0) << " dim = " << (void *)buf->dim << " = {\n"; for (int i = 0; i < buf->dimensions; i++) { @@ -1554,7 +1553,7 @@ void dump_wasmbuf(const Local &context, wasm32_ptr_t buf_ptr, const std } debug(0) << " }\n"; debug(0) << " flags = " << buf->flags << "\n"; - debug(0) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "," << buf->type.lanes << "\n"; + debug(0) << " type = " << (int)buf->type.code << "," << (int)buf->type.bits << "\n"; debug(0) << " dimensions = " << buf->dimensions << "\n"; debug(0) << " dim = " << buf->dim << " -> " << (void *)dim << " = {\n"; for (int i = 0; i < buf->dimensions; i++) { @@ -1814,9 +1813,8 @@ void wasm_jit_halide_trace_helper_callback(const v8::FunctionCallbackInfo #include #include +#include #include #else #include @@ -510,11 +511,13 @@ typedef enum halide_type_code_t #endif #endif -/** A runtime tag for a type in the halide type system. Can be ints, - * unsigned ints, or floats of various bit-widths (the 'bits' - * field). Can also be vectors of the same (by setting the 'lanes' - * field to something larger than one). This struct should be - * exactly 32-bits in size. */ +/** A runtime tag for an element type in the halide type system. Can be + * ints, unsigned ints, or floats of various bit-widths (the 'bits' field), + * or an opaque handle. + * + * This is the ABI/wire type. Unlike the compiler-side Halide::Type, it does + * NOT carry a vector width ('lanes'): a value crossing the ABI boundary is + * always a single element. */ struct halide_type_t { /** The basic type code: signed integer, unsigned integer, or floating point. */ #if (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) @@ -529,35 +532,27 @@ struct halide_type_t { HALIDE_ATTRIBUTE_ALIGN(1) uint8_t bits; - /** How many elements in a vector. This is 1 for scalar types. */ + /** Reserved for future element-kind payloads. */ HALIDE_ATTRIBUTE_ALIGN(2) - uint16_t lanes; + uint16_t reserved; #if (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) - /** Construct a runtime representation of a Halide type from: + /** Construct a runtime representation of a Halide element type from: * code: The fundamental type from an enum. - * bits: The bit size of one element. - * lanes: The number of vector elements in the type. */ - HALIDE_ALWAYS_INLINE constexpr halide_type_t(halide_type_code_t code, uint8_t bits, uint16_t lanes = 1) - : code(code), bits(bits), lanes(lanes) { + * bits: The bit size of one element. */ + HALIDE_ALWAYS_INLINE constexpr halide_type_t(halide_type_code_t code, uint8_t bits) + : code(code), bits(bits), reserved(0) { } /** Default constructor is required e.g. to declare halide_trace_event * instances. */ HALIDE_ALWAYS_INLINE constexpr halide_type_t() - : code((halide_type_code_t)0), bits(0), lanes(0) { - } - - HALIDE_ALWAYS_INLINE constexpr halide_type_t with_lanes(uint16_t new_lanes) const { - return halide_type_t((halide_type_code_t)code, bits, new_lanes); + : code((halide_type_code_t)0), bits(0), reserved(0) { } - HALIDE_ALWAYS_INLINE constexpr halide_type_t element_of() const { - return with_lanes(1); - } /** Compare two types for equality. */ HALIDE_ALWAYS_INLINE constexpr bool operator==(const halide_type_t &other) const { - return as_u32() == other.as_u32(); + return static_cast(*this) == static_cast(other); } HALIDE_ALWAYS_INLINE constexpr bool operator!=(const halide_type_t &other) const { @@ -565,22 +560,22 @@ struct halide_type_t { } HALIDE_ALWAYS_INLINE constexpr bool operator<(const halide_type_t &other) const { - return as_u32() < other.as_u32(); + return static_cast(*this) < static_cast(other); } - /** Size in bytes for a single element, even if width is not 1, of this type. */ + /** Size in bytes for a single element of this type. */ HALIDE_ALWAYS_INLINE constexpr int bytes() const { return (bits + 7) / 8; } - HALIDE_ALWAYS_INLINE constexpr uint32_t as_u32() const { + HALIDE_ALWAYS_INLINE constexpr operator uint32_t() const { // Note that this produces a result that is identical to memcpy'ing 'this' // into a u32 (on a little-endian machine, anyway), and at -O1 or greater // on Clang, the compiler knows this and optimizes this into a single 32-bit move. // (At -O0 it will look awful.) return static_cast(code) | - (static_cast(bits) << 8) | - (static_cast(lanes) << 16); + static_cast(bits) << 8 | + static_cast(reserved) << 16; } #endif }; @@ -630,10 +625,15 @@ struct halide_trace_event_t { */ const char *trace_tag; - /** If the event type is a load or a store, this is the type of - * the data. Otherwise, the value is meaningless. */ + /** If the event type is a load or a store, this is the (scalar) element + * type of the data. Otherwise, the value is meaningless. */ struct halide_type_t type; + /** For loads and stores, the number of vector lanes of the access (1 for a + * scalar access). The value pointed to by `value` is `lanes` elements of + * `type`. */ + int32_t lanes; + /** The type of event */ enum halide_trace_event_code_t event; @@ -709,7 +709,7 @@ extern halide_trace_t halide_set_custom_trace(halide_trace_t trace); struct halide_trace_packet_t { #ifdef __cplusplus HALIDE_ALWAYS_INLINE halide_trace_packet_t() - : size(0), event(halide_trace_load), parent_id(0), id(0), type(), dimensions(0) { + : size(0), event(halide_trace_load), parent_id(0), id(0), type_code((halide_type_code_t)0), type_bits(0), lanes(0), dimensions(0) { } #endif @@ -737,7 +737,16 @@ struct halide_trace_packet_t { int32_t value_index; }; union { - struct halide_type_t type; + struct { + /** The (scalar) element type code of the access (see halide_trace_event_t). + * Unpacked from halide_type_t rather than storing it directly, since + * halide_type_t's `reserved` field carries no meaning here. */ + uint8_t type_code; + /** The bit-width of the element type of the access. */ + uint8_t type_bits; + /** The number of vector lanes of the access (see halide_trace_event_t). */ + uint16_t lanes; + }; int32_t thread_id; }; int32_t dimensions; @@ -766,11 +775,17 @@ struct halide_trace_packet_t { return (void *)(coordinates() + dimensions); } + /** Reconstruct the (scalar) element type of the access from type_code and + * type_bits. Only meaningful for load and store events. */ + HALIDE_ALWAYS_INLINE struct halide_type_t type() const { + return halide_type_t((halide_type_code_t)type_code, type_bits); + } + /** Get the size of the value payload. Only load and store packets have a * value, so type is ignored for all other event types. */ HALIDE_ALWAYS_INLINE uint32_t value_bytes() const { if (event == halide_trace_load || event == halide_trace_store) { - return type.lanes * type.bytes(); + return lanes * type().bytes(); } return 0; } @@ -2306,6 +2321,23 @@ struct ArgumentInfo { } // namespace HalideFunctionInfo +inline std::ostream &operator<<(std::ostream &os, const halide_type_t &type) { + switch (type.code) { + case halide_type_int: + return os << "int" << (int)type.bits; + case halide_type_uint: + return type.bits ? os << "uint" << (int)type.bits : + os << "bool"; + case halide_type_float: + return os << "float" << (int)type.bits; + case halide_type_handle: + return os << "(void*)"; + case halide_type_bfloat: + return os << "bfloat" << (int)type.bits; + } + return os; +} + #endif // COMPILING_HALIDE_RUNTIME #endif // (__cplusplus >= 201103L || _MSVC_LANG >= 201103L) diff --git a/src/runtime/d3d12compute.cpp b/src/runtime/d3d12compute.cpp index 6fc1bac5834d..0fcf58cf4e37 100644 --- a/src/runtime/d3d12compute.cpp +++ b/src/runtime/d3d12compute.cpp @@ -416,7 +416,6 @@ WEAK DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_t }; halide_abort_if_false(user_context, (type.code >= 0) && (type.code <= 2)); - halide_abort_if_false(user_context, (type.lanes > 0) && (type.lanes <= 4)); int i = 0; switch (type.bytes()) { @@ -435,7 +434,7 @@ WEAK DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_t } DXGI_FORMAT format = DXGI_FORMAT_UNKNOWN; - format = FORMATS[(int)type.code][type.lanes - 1][i]; + format = FORMATS[(int)type.code][0][i]; return format; } @@ -694,17 +693,16 @@ WEAK size_t number_of_elements(void *user_context, const halide_buffer_t *buffer size_t element_size = 1; element_size *= buffer->type.bytes(); - element_size *= buffer->type.lanes; halide_abort_if_false(user_context, (element_size > 0)); size_t elements = size_in_bytes / element_size; halide_abort_if_false(user_context, (size_in_bytes % element_size) == 0); // 64-bit types are represented as pairs of uint32 elements in the UAV - // (there is no DXGI format for 64-bit scalars). Each 64-bit lane occupies - // two R32_UINT slots, so multi-lane 64-bit elements need 2 * lanes slots. + // (there is no DXGI format for 64-bit scalars). Each 64-bit element + // occupies two R32_UINT slots. if (is_64bit_type(buffer->type)) { - elements *= 2 * (size_t)buffer->type.lanes; + elements *= 2; } return elements; @@ -3308,10 +3306,9 @@ WEAK int halide_d3d12compute_image_device_malloc(void *user_context, halide_buff return halide_error_code_device_malloc_failed; } - // Only scalar (1-lane) element types are supported for textures - halide_type_t scalar_type = buf->type; - scalar_type.lanes = 1; - DXGI_FORMAT fmt = FindD3D12FormatForHalideType(user_context, scalar_type); + // Buffer element types are always scalar at the ABI (halide_type_t has no + // lanes), so the element type can be used directly. + DXGI_FORMAT fmt = FindD3D12FormatForHalideType(user_context, buf->type); if (fmt == DXGI_FORMAT_UNKNOWN) { error(user_context) << "D3D12Compute: unsupported element type for texture: " << buf->type; return halide_error_code_device_malloc_failed; @@ -3852,7 +3849,6 @@ WEAK int halide_d3d12compute_run(void *user_context, halide_type_t arg_type = arg_types[i]; arg_sizes[i] = arg_type.bytes(); halide_abort_if_false(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); - halide_abort_if_false(user_context, arg_type.lanes == 1); halide_abort_if_false(user_context, arg_sizes[i] > 0); halide_abort_if_false(user_context, arg_sizes[i] <= 8); // Sub-32-bit scalars are widened to 32-bit in the cbuffer layout; @@ -4245,10 +4241,10 @@ WEAK int d3d12compute_device_crop_from_offset(void *user_context, // DXGI format already encodes the lane count, so the scale is 1. int64_t uav_offset = offset; if (is_64bit_type(dst->type)) { - uav_offset *= 2 * (int64_t)dst->type.lanes; + uav_offset *= 2; } new_handle->offset = old_handle->offset + uav_offset; - new_handle->offsetInBytes = (old_handle->offset + offset) * dst->type.bytes() * dst->type.lanes; + new_handle->offsetInBytes = (old_handle->offset + offset) * dst->type.bytes(); // for some reason, 'dst->number_of_elements()' is always returning 1 // later on when 'set_input()' is called... new_handle->elements = old_handle->elements - uav_offset; diff --git a/src/runtime/halide_buffer_t.cpp b/src/runtime/halide_buffer_t.cpp index d4a80e6470c8..a3bf1e68f9b2 100644 --- a/src/runtime/halide_buffer_t.cpp +++ b/src/runtime/halide_buffer_t.cpp @@ -88,7 +88,7 @@ bool _halide_buffer_is_bounds_query(const halide_buffer_t *buf) { HALIDE_BUFFER_HELPER_ATTRS uint32_t _halide_buffer_get_type(const halide_buffer_t *buf) { - return buf->type.as_u32(); + return buf->type; } HALIDE_BUFFER_HELPER_ATTRS @@ -104,9 +104,7 @@ halide_buffer_t *_halide_buffer_init(halide_buffer_t *dst, dst->host = (uint8_t *)host; dst->device = device; dst->device_interface = device_interface; - dst->type.code = (halide_type_code_t)type_code; - dst->type.bits = (uint8_t)type_bits; - dst->type.lanes = 1; + dst->type = {(halide_type_code_t)type_code, (uint8_t)type_bits}; dst->dimensions = dimensions; dst->dim = dst_shape; if (shape != dst->dim) { diff --git a/src/runtime/metal.cpp b/src/runtime/metal.cpp index 1d9d7d7edd08..95bbf31a8f29 100644 --- a/src/runtime/metal.cpp +++ b/src/runtime/metal.cpp @@ -957,7 +957,7 @@ WEAK int halide_metal_run(void *user_context, set_compute_pipeline_state(encoder, pipeline_state); int num_kernel_args = 0; - for (int i = 0; arg_types[i].as_u32() != 0; i++) { + for (int i = 0; static_cast(arg_types[i]) != 0; i++) { ++num_kernel_args; } diff --git a/src/runtime/to_string.cpp b/src/runtime/to_string.cpp index 1200ca5c07d9..7bb36d76d4cd 100644 --- a/src/runtime/to_string.cpp +++ b/src/runtime/to_string.cpp @@ -287,10 +287,6 @@ WEAK char *halide_type_to_string(char *dst, char *end, const halide_type_t *t) { } dst = halide_string_to_string(dst, end, code_name); dst = halide_uint64_to_string(dst, end, t->bits, 1); - if (t->lanes != 1) { - dst = halide_string_to_string(dst, end, "x"); - dst = halide_uint64_to_string(dst, end, t->lanes, 1); - } return dst; } diff --git a/src/runtime/trace_helper.cpp b/src/runtime/trace_helper.cpp index 9e968756ebd6..7052190eb592 100644 --- a/src/runtime/trace_helper.cpp +++ b/src/runtime/trace_helper.cpp @@ -16,9 +16,8 @@ WEAK int halide_trace_helper(void *user_context, event.value = value; event.coordinates = coords; event.trace_tag = trace_tag; - event.type.code = (halide_type_code_t)type_code; - event.type.bits = (uint8_t)type_bits; - event.type.lanes = (uint16_t)type_lanes; + event.type = {(halide_type_code_t)type_code, (uint8_t)type_bits}; + event.lanes = type_lanes; event.event = (halide_trace_event_code_t)code; event.parent_id = parent_id; event.thread_id = thread_id; diff --git a/src/runtime/tracing.cpp b/src/runtime/tracing.cpp index bdc639dd683b..ca9713a66f09 100644 --- a/src/runtime/tracing.cpp +++ b/src/runtime/tracing.cpp @@ -187,7 +187,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t if (fd > 0) { // Compute the total packet size uint32_t value_bytes = (e->event == halide_trace_load || e->event == halide_trace_store) ? - (uint32_t)(e->type.lanes * e->type.bytes()) : + (uint32_t)(e->lanes * e->type.bytes()) : 0; uint32_t header_bytes = (uint32_t)sizeof(halide_trace_packet_t); uint32_t coords_bytes = e->dimensions * (uint32_t)sizeof(int32_t); @@ -210,7 +210,9 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t packet->dimensions = e->dimensions; if (e->event == halide_trace_load || e->event == halide_trace_store) { packet->value_index = e->value_index; - packet->type = e->type; + packet->type_code = (uint8_t)e->type.code; + packet->type_bits = e->type.bits; + packet->lanes = (uint16_t)e->lanes; } else { packet->id = my_id; packet->thread_id = (e->event == halide_trace_begin_parallel_task) ? e->thread_id : 0; @@ -262,12 +264,12 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t bool print_value = (e->event < 2); ss << event_types[e->event] << " " << e->func << "." << e->value_index << "("; - if (e->type.lanes > 1) { + if (e->lanes > 1) { ss << "<"; } for (int i = 0; i < e->dimensions; i++) { if (i > 0) { - if ((e->type.lanes > 1) && (i % e->type.lanes) == 0) { + if ((e->lanes > 1) && (i % e->lanes) == 0) { ss << ">, <"; } else { ss << ", "; @@ -275,19 +277,19 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t } ss << e->coordinates[i]; } - if (e->type.lanes > 1) { + if (e->lanes > 1) { ss << ">)"; } else { ss << ")"; } if (print_value) { - if (e->type.lanes > 1) { + if (e->lanes > 1) { ss << " = <"; } else { ss << " = "; } - for (int i = 0; i < e->type.lanes; i++) { + for (int i = 0; i < e->lanes; i++) { if (i > 0) { ss << ", "; } @@ -324,7 +326,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t ss << ((void **)(e->value))[i]; } } - if (e->type.lanes > 1) { + if (e->lanes > 1) { ss << ">"; } } diff --git a/src/runtime/webgpu.cpp b/src/runtime/webgpu.cpp index b57113db3d0e..5b1f4d930fc9 100644 --- a/src/runtime/webgpu.cpp +++ b/src/runtime/webgpu.cpp @@ -1007,7 +1007,6 @@ WEAK int halide_webgpu_run(void *user_context, } halide_type_t arg_type = arg_types[a]; - halide_debug_assert(user_context, arg_type.lanes == 1); halide_debug_assert(user_context, arg_type.bits > 0); halide_debug_assert(user_context, arg_type.bits <= 32); diff --git a/test/correctness/CMakeLists.txt b/test/correctness/CMakeLists.txt index 821a62525a2d..241dba9db8a0 100644 --- a/test/correctness/CMakeLists.txt +++ b/test/correctness/CMakeLists.txt @@ -199,6 +199,7 @@ tests( intrinsics.cpp invalid_gpu_loop_nests.cpp inverse.cpp + irmatch.cpp irprinter.cpp isnan.cpp issue_3926.cpp diff --git a/test/correctness/compute_with.cpp b/test/correctness/compute_with.cpp index 177bfd53cec4..2bcf12931415 100644 --- a/test/correctness/compute_with.cpp +++ b/test/correctness/compute_with.cpp @@ -70,7 +70,7 @@ int my_trace(JITUserContext *user_context, const halide_trace_event_t *e) { const auto &iter = stores.find(fname); if (iter != stores.end()) { const Bound &b = iter->second; - if (!check_coordinates(b, e->coordinates, e->dimensions, e->type.lanes, "store", fname)) { + if (!check_coordinates(b, e->coordinates, e->dimensions, e->lanes, "store", fname)) { exit(1); } } @@ -80,7 +80,7 @@ int my_trace(JITUserContext *user_context, const halide_trace_event_t *e) { const auto &iter = loads.find(fname); if (iter != loads.end()) { const Bound &b = iter->second; - if (!check_coordinates(b, e->coordinates, e->dimensions, e->type.lanes, "load", fname)) { + if (!check_coordinates(b, e->coordinates, e->dimensions, e->lanes, "load", fname)) { exit(1); } } diff --git a/test/correctness/irmatch.cpp b/test/correctness/irmatch.cpp new file mode 100644 index 000000000000..3df2f3ce2add --- /dev/null +++ b/test/correctness/irmatch.cpp @@ -0,0 +1,67 @@ +#include "Halide.h" + +#include + +using namespace Halide; +using namespace Halide::Internal; + +namespace { + +using IRMatcher::Intrin; +using IRMatcher::pattern_arg; + +template +auto make_struct(A &&a, B &&b) noexcept -> Intrin { + return {pattern_arg(a), pattern_arg(b)}; +} + +template +auto load_typed_struct_member(A &&a, B &&b, C &&c) noexcept -> Intrin { + return {pattern_arg(a), pattern_arg(b), pattern_arg(c)}; +} + +} // namespace + +struct Foo; +HALIDE_DECLARE_EXTERN_STRUCT_TYPE(Foo); + +int main(int argc, char **argv) { + const Type foo_ptr = type_of(); + + // A struct that stores a void* in its first field. + Expr stored0 = Variable::make(Handle(), "raw_ptr"); + Expr stored1 = Variable::make(Int(32), "other"); + Expr inst = Call::make(Handle(), Call::make_struct, {stored0, stored1}, Call::PureIntrinsic); + Expr proto = Variable::make(Handle(), "closure_prototype"); + + // Load it back as a Foo*: load_typed_struct_member's job is to impose the + // member's declared type on the untyped slot. + Expr op = Call::make(foo_ptr, Call::load_typed_struct_member, {inst, proto, 0}, Call::Intrinsic); + + IRMatcher::Wild<0> v0; + IRMatcher::Wild<1> v1; + IRMatcher::Wild<2> pr; + + // Textbook SROA rule instance. + auto rewrite = IRMatcher::rewriter(op, op.type()); + bool matched = rewrite(load_typed_struct_member(make_struct(v0, v1), pr, 0), + IRMatcher::cast(op.type(), v0)); + + if (!matched) { + std::cout << "Rewrite pattern failed to match.\n"; + return 1; + } + + const Type result_type = rewrite.result.type(); + if (!foo_ptr.same_handle_type(result_type)) { + std::cout << "Initial: " << op << "\n" + << "Result : " << rewrite.result << "\n" + << "Type information was lost!\n" + << " expected: " << foo_ptr << "\n" + << " got: " << result_type << "\n"; + return 1; + } + + std::cout << "Success!\n"; + return 0; +} diff --git a/test/correctness/reschedule.cpp b/test/correctness/reschedule.cpp index 2f591a29724b..b107082a3738 100644 --- a/test/correctness/reschedule.cpp +++ b/test/correctness/reschedule.cpp @@ -9,7 +9,7 @@ bool vector_store = false, scalar_store = false; int my_trace(JITUserContext *user_context, const halide_trace_event_t *ev) { if (ev->event == halide_trace_store) { - if (ev->type.lanes > 1) { + if (ev->lanes > 1) { vector_store = true; } else { scalar_store = true; diff --git a/test/correctness/simd_op_check.h b/test/correctness/simd_op_check.h index 828b16dc969b..77f452765ebc 100644 --- a/test/correctness/simd_op_check.h +++ b/test/correctness/simd_op_check.h @@ -371,7 +371,7 @@ class SimdOpCheckTest { std::vector args(image_params.size() + 1); for (size_t i = 0; i < image_params.size(); i++) { args[i] = image_params[i]; - inputs[i] = Runtime::Buffer<>(args[i].type, nullptr, 0); + inputs[i] = Runtime::Buffer<>(args[i].type.to_abi(), nullptr, 0); } args.back() = rows; diff --git a/test/correctness/simplify.cpp b/test/correctness/simplify.cpp index 839769231c98..42dbdb480961 100644 --- a/test/correctness/simplify.cpp +++ b/test/correctness/simplify.cpp @@ -2104,30 +2104,30 @@ void check_overflow() { template void check_clz(uint64_t value, uint64_t result) { - Expr x = Variable::make(halide_type_of(), "x"); + Expr x = Variable::make(type_of(), "x"); check(Let::make("x", cast(Expr(value)), count_leading_zeros(x)), cast(Expr(result))); - Type vt = halide_type_of().with_lanes(4); + Type vt = type_of().with_lanes(4); Expr xv = Variable::make(vt, "x"); check(Let::make("x", cast(vt, broadcast(Expr(value), 4)), count_leading_zeros(xv)), cast(vt, broadcast(Expr(result), 4))); } template void check_ctz(uint64_t value, uint64_t result) { - Expr x = Variable::make(halide_type_of(), "x"); + Expr x = Variable::make(type_of(), "x"); check(Let::make("x", cast(Expr(value)), count_trailing_zeros(x)), cast(Expr(result))); - Type vt = halide_type_of().with_lanes(4); + Type vt = type_of().with_lanes(4); Expr xv = Variable::make(vt, "x"); check(Let::make("x", cast(vt, broadcast(Expr(value), 4)), count_trailing_zeros(xv)), cast(vt, broadcast(Expr(result), 4))); } template void check_popcount(uint64_t value, uint64_t result) { - Expr x = Variable::make(halide_type_of(), "x"); + Expr x = Variable::make(type_of(), "x"); check(Let::make("x", cast(Expr(value)), popcount(x)), cast(Expr(result))); - Type vt = halide_type_of().with_lanes(4); + Type vt = type_of().with_lanes(4); Expr xv = Variable::make(vt, "x"); check(Let::make("x", cast(vt, broadcast(Expr(value), 4)), popcount(xv)), cast(vt, broadcast(Expr(result), 4))); } diff --git a/test/correctness/specialize.cpp b/test/correctness/specialize.cpp index dff81748aebc..82cafa183f35 100644 --- a/test/correctness/specialize.cpp +++ b/test/correctness/specialize.cpp @@ -16,9 +16,9 @@ void reset_trace() { int my_trace(JITUserContext *user_context, const halide_trace_event_t *ev) { if (ev->event == halide_trace_store) { - if (ev->type.lanes > 1) { + if (ev->lanes > 1) { vector_store = true; - vector_store_lanes = ev->type.lanes; + vector_store_lanes = ev->lanes; } else { scalar_store = true; } diff --git a/test/correctness/tracing.cpp b/test/correctness/tracing.cpp index fd5c401489ca..42e6fd9b24e5 100644 --- a/test/correctness/tracing.cpp +++ b/test/correctness/tracing.cpp @@ -92,7 +92,7 @@ bool events_match(const event &a, const event &b) { } int my_trace(JITUserContext *, const halide_trace_event_t *ev) { - assert(ev->dimensions <= 4 && ev->type.lanes <= 4); + assert(ev->dimensions <= 4 && ev->lanes <= 4); // Record this event in the trace array event e{}; @@ -101,13 +101,13 @@ int my_trace(JITUserContext *, const halide_trace_event_t *ev) { e.event_type = ev->event; e.type_code = ev->type.code; e.bits = ev->type.bits; - e.width = ev->type.lanes; + e.width = ev->lanes; e.value_index = ev->value_index; e.num_int_args = ev->dimensions; for (int i = 0; i < ev->dimensions; i++) { e.int_args[i] = ev->coordinates[i]; } - for (int i = 0; i < ev->type.lanes; i++) { + for (int i = 0; i < ev->lanes; i++) { e.value[i] = static_cast(ev->value)[i]; } if (ev->trace_tag) { diff --git a/test/correctness/tracing_broadcast.cpp b/test/correctness/tracing_broadcast.cpp index 2950d1b54876..fe7f045495e4 100644 --- a/test/correctness/tracing_broadcast.cpp +++ b/test/correctness/tracing_broadcast.cpp @@ -5,7 +5,7 @@ using namespace Halide; int my_trace(JITUserContext *user_context, const halide_trace_event_t *e) { if (e->event == halide_trace_store) { - for (int i = 0; i < e->type.lanes; ++i) { + for (int i = 0; i < e->lanes; ++i) { int val = ((const int *)(e->value))[i]; if (val != 1234567890) { printf("All values stored should have been 1234567890\n" diff --git a/test/correctness/tracing_thread_ids.cpp b/test/correctness/tracing_thread_ids.cpp index 62f802e54ae1..f48da0964480 100644 --- a/test/correctness/tracing_thread_ids.cpp +++ b/test/correctness/tracing_thread_ids.cpp @@ -44,9 +44,9 @@ int my_trace(JITUserContext *, const halide_trace_event_t *ev) { if (ev->event == halide_trace_begin_parallel_task) { parallel_task_ids.insert(id); } else if (ev->event == halide_trace_store) { - if (ev->dimensions != 2 || ev->type.lanes != 1) { + if (ev->dimensions != 2 || ev->lanes != 1) { printf("Store trace event had %d dimensions and %d lanes.\n", - ev->dimensions, ev->type.lanes); + ev->dimensions, ev->lanes); trace_failed = true; return id; } diff --git a/test/correctness/vectorize_guard_with_if.cpp b/test/correctness/vectorize_guard_with_if.cpp index dad6d0fa208c..8e5f4289f605 100644 --- a/test/correctness/vectorize_guard_with_if.cpp +++ b/test/correctness/vectorize_guard_with_if.cpp @@ -6,7 +6,7 @@ int num_vector_stores = 0; int num_scalar_stores = 0; int my_trace(JITUserContext *user_context, const halide_trace_event_t *e) { if (e->event == halide_trace_store) { - if (e->type.lanes > 1) { + if (e->lanes > 1) { num_vector_stores++; } else { num_scalar_stores++; diff --git a/test/generator/metadata_tester_aottest.cpp b/test/generator/metadata_tester_aottest.cpp index c53d1e61945f..e15e1a77a92f 100644 --- a/test/generator/metadata_tester_aottest.cpp +++ b/test/generator/metadata_tester_aottest.cpp @@ -15,21 +15,7 @@ namespace { using namespace Halide::Runtime; -const int kSize = 32; - -inline std::ostream &operator<<(std::ostream &o, const halide_type_t &type) { - if (type.code == halide_type_uint && type.bits == 1) { - o << "bool"; - } else { - assert(type.code >= 0 && type.code <= 3); - static const char *const names[4] = {"int", "uint", "float", "handle"}; - o << names[type.code] << (int)type.bits; - } - if (type.lanes > 1) { - o << "x" << (int)type.lanes; - } - return o; -} +constexpr int kSize = 32; struct typed_scalar { halide_type_t type; @@ -44,30 +30,30 @@ struct typed_scalar { std::cerr << "Mismatched types\n"; exit(1); } - switch (type.element_of().as_u32()) { - case halide_type_t(halide_type_float, 32).as_u32(): + switch (type) { + case halide_type_t(halide_type_float, 32): return value.u.f32 == that.value.u.f32; - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return value.u.f64 == that.value.u.f64; - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): return value.u.i8 == that.value.u.i8; - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return value.u.i16 == that.value.u.i16; - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return value.u.i32 == that.value.u.i32; - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return value.u.i64 == that.value.u.i64; - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return value.u.b == that.value.u.b; - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return value.u.u8 == that.value.u.u8; - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return value.u.u16 == that.value.u.u16; - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return value.u.u32 == that.value.u.u32; - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return value.u.u64 == that.value.u.u64; - case halide_type_t(halide_type_handle, 64).as_u32(): + case halide_type_t(halide_type_handle, 64): return value.u.handle == that.value.u.handle; default: std::cerr << "Unsupported type\n"; @@ -81,41 +67,41 @@ struct typed_scalar { } friend std::ostream &operator<<(std::ostream &o, const typed_scalar &s) { - switch (s.type.element_of().as_u32()) { - case halide_type_t(halide_type_float, 32).as_u32(): + switch (s.type) { + case halide_type_t(halide_type_float, 32): o << s.value.u.f32; break; - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): o << s.value.u.f64; break; - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): o << (int)s.value.u.i8; break; - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): o << s.value.u.i16; break; - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): o << s.value.u.i32; break; - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): o << s.value.u.i64; break; - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): o << (s.value.u.b ? "true" : "false"); break; - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): o << (int)s.value.u.u8; break; - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): o << s.value.u.u16; break; - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): o << s.value.u.u32; break; - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): o << s.value.u.u64; break; - case halide_type_t(halide_type_handle, 64).as_u32(): + case halide_type_t(halide_type_handle, 64): o << (uint64_t)s.value.u.handle; break; default: @@ -1364,11 +1350,11 @@ constexpr char arginfo_to_sigchar(::HalideFunctionInfo::ArgumentInfo arg) { return '#'; } else { -#define HANDLE_CASE(CODE, BITS, CHAR) \ - case halide_type_t(CODE, BITS).as_u32(): \ +#define HANDLE_CASE(CODE, BITS, CHAR) \ + case halide_type_t(CODE, BITS): \ return (CHAR); - switch (arg.type.as_u32()) { + switch (arg.type) { HANDLE_CASE(halide_type_bfloat, 16, '!') HANDLE_CASE(halide_type_float, 16, 'e') HANDLE_CASE(halide_type_float, 32, 'f') @@ -1413,7 +1399,7 @@ int main(int argc, char **argv) { Buffer input = make_image(); Buffer input_array[2] = {make_image(), make_image()}; // TODO: there is no runtime type for float16, so we'll declare this using a halide_type_t - const halide_type_t halide_type_float16 = halide_type_t(halide_type_float, 16, 1); + const halide_type_t halide_type_float16 = halide_type_t(halide_type_float, 16); Buffer<> input_f16 = Buffer<>(halide_type_float16, kSize); Buffer output0(kSize, kSize, 3); diff --git a/test/generator/stubtest_jittest.cpp b/test/generator/stubtest_jittest.cpp index 685b717a2914..0ae9aaaea7de 100644 --- a/test/generator/stubtest_jittest.cpp +++ b/test/generator/stubtest_jittest.cpp @@ -130,8 +130,8 @@ int main(int argc, char **argv) { Buffer array_buffer_input1 = make_image(1); Buffer simple_output(kSize, kSize, 3); // TODO: see Issues #3709, #3967 - Buffer float16_output(halide_type_t(halide_type_float, 16), kSize, kSize, 3); - Buffer bfloat16_output(halide_type_t(halide_type_bfloat, 16), kSize, kSize, 3); + Buffer float16_output(Float(16), kSize, kSize, 3); + Buffer bfloat16_output(BFloat(16), kSize, kSize, 3); Buffer tuple_output0(kSize, kSize, 3), tuple_output1(kSize, kSize, 3); Buffer array_output0(kSize, kSize, 3), array_output1(kSize, kSize, 3); Buffer static_compiled_buffer_output(kSize, kSize, 3); @@ -205,8 +205,8 @@ int main(int argc, char **argv) { Buffer array_buffer_input1 = make_image(1); Buffer simple_output(kSize, kSize, 3); // TODO: see Issues #3709, #3967 - Buffer float16_output(halide_type_t(halide_type_float, 16), kSize, kSize, 3); - Buffer bfloat16_output(halide_type_t(halide_type_bfloat, 16), kSize, kSize, 3); + Buffer float16_output(Float(16), kSize, kSize, 3); + Buffer bfloat16_output(BFloat(16), kSize, kSize, 3); Buffer tuple_output0(kSize, kSize, 3), tuple_output1(kSize, kSize, 3); Buffer array_output0(kSize, kSize, 3), array_output1(kSize, kSize, 3); Buffer static_compiled_buffer_output(kSize, kSize, 3); diff --git a/tools/RunGen.h b/tools/RunGen.h index 2422fa93a6a1..d53f605fc390 100644 --- a/tools/RunGen.h +++ b/tools/RunGen.h @@ -34,21 +34,6 @@ using Shape = std::vector; // estimate is provided for the input in question. using ShapePromise = std::function; -// Standard stream output for halide_type_t -inline std::ostream &operator<<(std::ostream &stream, const halide_type_t &type) { - if (type.code == halide_type_uint && type.bits == 1) { - stream << "bool"; - } else { - assert(type.code >= 0 && type.code <= 3); - static const char *const names[4] = {"int", "uint", "float", "handle"}; - stream << names[type.code] << (int)type.bits; - } - if (type.lanes > 1) { - stream << "x" << (int)type.lanes; - } - return stream; -} - // Standard stream output for halide_dimension_t inline std::ostream &operator<<(std::ostream &stream, const halide_dimension_t &d) { stream << "[" << d.min << "," << d.extent << "," << d.stride << "]"; @@ -200,11 +185,11 @@ inline std::vector split_string(const std::string &source, template class Functor, typename... Args> auto dynamic_type_dispatch(const halide_type_t &type, Args &&...args) -> decltype(std::declval>()(std::forward(args)...)) { -#define HANDLE_CASE(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): \ +#define HANDLE_CASE(CODE, BITS, TYPE) \ + case halide_type_t(CODE, BITS): \ return Functor()(std::forward(args)...); - switch (type.element_of().as_u32()) { + switch (type) { HANDLE_CASE(halide_type_float, 32, float) HANDLE_CASE(halide_type_float, 64, double) HANDLE_CASE(halide_type_int, 8, int8_t) @@ -581,28 +566,28 @@ struct FillWithScalar { template::value>::type * = nullptr> T as_T(const halide_scalar_value_t &value) { constexpr halide_type_t type = halide_type_of(); - switch (type.element_of().as_u32()) { - case halide_type_t(halide_type_int, 8).as_u32(): + switch (type) { + case halide_type_t(halide_type_int, 8): return (T)value.u.i8; - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return (T)value.u.i16; - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return (T)value.u.i32; - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return (T)value.u.i64; - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return (T)value.u.b; - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return (T)value.u.u8; - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return (T)value.u.u16; - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return (T)value.u.u32; - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return (T)value.u.u64; - case halide_type_t(halide_type_float, 32).as_u32(): + case halide_type_t(halide_type_float, 32): return (T)value.u.f32; - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return (T)value.u.f64; default: fail() << "Can't convert value with type: " << (int)type.code << "bits: " << type.bits; @@ -613,8 +598,8 @@ struct FillWithScalar { template::value>::type * = nullptr> T as_T(const halide_scalar_value_t &value) { constexpr halide_type_t type = halide_type_of(); - switch (type.element_of().as_u32()) { - case halide_type_t(halide_type_handle, 64).as_u32(): + switch (type) { + case halide_type_t(halide_type_handle, 64): return (T)value.u.handle; default: fail() << "Can't convert value with type: " << (int)type.code << "bits: " << type.bits; @@ -653,41 +638,41 @@ inline Halide::Tools::FormatInfo best_save_format(const Buffer<> &b, const std:: inline std::string scalar_to_string(const halide_type_t &type, const halide_scalar_value_t &value) { std::ostringstream o; - switch (type.element_of().as_u32()) { - case halide_type_t(halide_type_float, 32).as_u32(): + switch (type) { + case halide_type_t(halide_type_float, 32): o << value.u.f32; break; - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): o << value.u.f64; break; - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): o << (int)value.u.i8; break; - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): o << value.u.i16; break; - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): o << value.u.i32; break; - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): o << value.u.i64; break; - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): o << (value.u.b ? "true" : "false"); break; - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): o << (int)value.u.u8; break; - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): o << value.u.u16; break; - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): o << value.u.u32; break; - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): o << value.u.u64; break; - case halide_type_t(halide_type_handle, 64).as_u32(): + case halide_type_t(halide_type_handle, 64): o << (uint64_t)value.u.handle; break; default: diff --git a/tools/halide_image_io.h b/tools/halide_image_io.h index 1f3f8dd01f99..9bcca42941a7 100644 --- a/tools/halide_image_io.h +++ b/tools/halide_image_io.h @@ -46,27 +46,21 @@ struct FormatInfo { int dimensions; bool operator<(const FormatInfo &other) const { - if (type.code < other.type.code) { - return true; - } else if (type.code > other.type.code) { - return false; - } - if (type.bits < other.type.bits) { - return true; - } else if (type.bits > other.type.bits) { - return false; - } - if (type.lanes < other.type.lanes) { - return true; - } else if (type.lanes > other.type.lanes) { - return false; - } - return (dimensions < other.dimensions); + return std::tie(type, dimensions) < std::tie(other.type, other.dimensions); } }; namespace Internal { +template +halide_type_t ensure_abi_type(T ty) { + if constexpr (std::is_same_v) { + return ty; + } else { + return ty.to_abi(); + } +} + typedef bool (*CheckFunc)(bool condition, const char *msg); inline bool CheckFail(bool condition, const char *msg) { @@ -995,7 +989,7 @@ bool save_png(ImageType &im, const std::string &filename) { png_init_io(png_ptr, f.f); - const halide_type_t im_type = im.type(); + const halide_type_t im_type = ensure_abi_type(im.type()); const int bit_depth = im_type.bits; png_set_IHDR(png_ptr, info_ptr, width, height, @@ -1106,7 +1100,7 @@ bool save_pnm(ImageType &im, const int channels, const std::string &filename) { return false; } - const halide_type_t im_type = im.type(); + const halide_type_t im_type = ensure_abi_type(im.type()); const int width = im.width(); const int height = im.height(); const int bit_depth = im_type.bits; @@ -1278,8 +1272,7 @@ struct NpyHeader { // any elements, and is in strictly planar order. template bool buffer_is_compact_planar(ImageType &im) { - const halide_type_t im_type = im.type(); - const size_t elem_size = (im_type.bits / 8); + const size_t elem_size = im.type().bytes(); if (((const uint8_t *)im.begin() + (im.number_of_elements() * elem_size)) != (const uint8_t *)im.end()) { return false; } @@ -1346,7 +1339,7 @@ bool load_npy(const std::string &filename, ImageType *im) { return false; } - halide_type_t im_type((halide_type_code_t)0, 0, 0); + halide_type_t im_type((halide_type_code_t)0, 0); for (const auto &d : npy_dtypes) { if (h.type_code == d.second.type_code && h.type_bytes == d.second.type_bytes) { im_type = d.first; @@ -1400,7 +1393,7 @@ bool save_npy(ImageType &im, const std::string &filename) { return false; } - const halide_type_t im_type = im.type(); + const halide_type_t im_type = ensure_abi_type(im.type()); npy_dtype_info_t di = {0, 0, 0}; for (const auto &d : npy_dtypes) { if (d.first == im_type) { @@ -2229,7 +2222,7 @@ bool save_tiff(ImageType &im, const std::string &filename) { shape[i].extent = 1; shape[i].stride = 0; } - const halide_type_t im_type = im.type(); + const halide_type_t im_type = ensure_abi_type(im.type()); if (!check(im_type.code >= 0 && im_type.code < 3, "Unsupported image type")) { return false; } @@ -2326,7 +2319,7 @@ bool save_tiff(ImageType &im, const std::string &filename) { // Otherwise, write it out via manual traversal. #define HANDLE_CASE(CODE, BITS, TYPE) \ - case halide_type_t(CODE, BITS).as_u32(): { \ + case halide_type_t(CODE, BITS): { \ ElemWriter ew(&f); \ im.template as().for_each_value(ew); \ if (!check(ew.ok, "TIFF write failed")) { \ @@ -2335,7 +2328,7 @@ bool save_tiff(ImageType &im, const std::string &filename) { break; \ } - switch (im_type.element_of().as_u32()) { + switch (im_type) { HANDLE_CASE(halide_type_float, 32, float) HANDLE_CASE(halide_type_float, 64, double) HANDLE_CASE(halide_type_int, 8, int8_t) @@ -2499,33 +2492,32 @@ struct ImageTypeConversion { "This variant of convert_image() requires a dynamically-typed image"); constexpr int AnyDims = Internal::AnyDims; - const halide_type_t src_type = src.type(); - switch (src_type.element_of().as_u32()) { + switch (Internal::ensure_abi_type(src.type())) { #if HALIDE_CPP_COMPILER_HAS_FLOAT16 - case halide_type_t(halide_type_float, 16).as_u32(): + case halide_type_t(halide_type_float, 16): return convert_image(src.template as<_Float16, AnyDims>()); #endif - case halide_type_t(halide_type_float, 32).as_u32(): + case halide_type_t(halide_type_float, 32): return convert_image(src.template as()); - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return convert_image(src.template as()); - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): return convert_image(src.template as()); - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return convert_image(src.template as()); - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return convert_image(src.template as()); - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return convert_image(src.template as()); - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return convert_image(src.template as()); - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return convert_image(src.template as()); - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return convert_image(src.template as()); - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return convert_image(src.template as()); - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return convert_image(src.template as()); default: assert(false && "Unsupported type"); @@ -2549,32 +2541,32 @@ struct ImageTypeConversion { // Call the appropriate static-to-static conversion routine // based on the desired dst type. - switch (dst_type.element_of().as_u32()) { + switch (dst_type) { #if HALIDE_CPP_COMPILER_HAS_FLOAT16 - case halide_type_t(halide_type_float, 16).as_u32(): + case halide_type_t(halide_type_float, 16): return convert_image<_Float16>(src); #endif - case halide_type_t(halide_type_float, 32).as_u32(): + case halide_type_t(halide_type_float, 32): return convert_image(src); - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return convert_image(src); - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): return convert_image(src); - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return convert_image(src); - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return convert_image(src); - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return convert_image(src); - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return convert_image(src); - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return convert_image(src); - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return convert_image(src); - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return convert_image(src); - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return convert_image(src); default: assert(false && "Unsupported type"); @@ -2601,29 +2593,28 @@ struct ImageTypeConversion { // and call the static-to-dynamic variant of this method. (Note that // this forces instantiation of the complete any-to-any conversion // matrix of code.) - const halide_type_t src_type = src.type(); - switch (src_type.element_of().as_u32()) { - case halide_type_t(halide_type_float, 32).as_u32(): + switch (Internal::ensure_abi_type(src.type())) { + case halide_type_t(halide_type_float, 32): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_int, 8).as_u32(): + case halide_type_t(halide_type_int, 8): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return convert_image(src.template as(), dst_type); - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return convert_image(src.template as(), dst_type); default: assert(false && "Unsupported type"); @@ -2672,7 +2663,8 @@ bool save(ImageType &im, const std::string &filename) { if (!Internal::find_imageio(filename, &imageio)) { return false; } - if (!check(imageio.query().count({im.type(), im.dimensions()}) > 0, "Image cannot be saved in this format")) { + auto type = Internal::ensure_abi_type(im.type()); + if (!check(imageio.query().count({type, im.dimensions()}) > 0, "Image cannot be saved in this format")) { return false; } diff --git a/tools/halide_trace_config.h b/tools/halide_trace_config.h index c720a097c81c..6f95d439f016 100644 --- a/tools/halide_trace_config.h +++ b/tools/halide_trace_config.h @@ -43,15 +43,15 @@ inline std::string unescape_spaces(const std::string &str) { return replace_all(str, "\\x20", " "); } -inline std::ostream &operator<<(std::ostream &os, const halide_type_t &t) { - os << (int)t.code << " " << (int)t.bits << " " << t.lanes; +inline std::ostream &write_halide_type(std::ostream &os, const halide_type_t &t) { + os << (int)t.code << " " << (int)t.bits; return os; } -inline std::istream &operator>>(std::istream &is, halide_type_t &t) { +inline std::istream &read_halide_type(std::istream &is, halide_type_t &t) { // type.code is an enum; type.bits is a uint8 and might be read as char. int type_code, type_bits; - is >> type_code >> type_bits >> t.lanes; + is >> type_code >> type_bits; t.code = (halide_type_code_t)type_code; t.bits = (uint8_t)type_bits; return is; @@ -66,6 +66,14 @@ std::ostream &operator<<(std::ostream &os, const std::vector &v) { return os; } +inline std::ostream &operator<<(std::ostream &os, const std::vector &v) { + os << v.size() << " "; + for (const halide_type_t &t : v) { + write_halide_type(os, t) << " "; + } + return os; +} + template std::istream &operator>>(std::istream &is, std::vector &v) { v.clear(); @@ -79,6 +87,18 @@ std::istream &operator>>(std::istream &is, std::vector &v) { return is; } +inline std::istream &operator>>(std::istream &is, std::vector &v) { + v.clear(); + size_t size; + is >> size; + for (size_t i = 0; i < size; ++i) { + halide_type_t tmp; + read_halide_type(is, tmp); + v.push_back(tmp); + } + return is; +} + struct Point { int x = 0, y = 0; @@ -512,9 +532,6 @@ struct FuncTypeAndDim { os << " types:"; for (const auto &type : types) { os << " " << type_name[type.code & 3] << (int)type.bits; - if (type.lanes > 1) { - os << "x" << type.lanes; - } } os << "\n"; os << " dims: " << dims << "\n"; diff --git a/util/HalideTraceDump.cpp b/util/HalideTraceDump.cpp index d7a5334641a1..03a1c298dbdc 100644 --- a/util/HalideTraceDump.cpp +++ b/util/HalideTraceDump.cpp @@ -48,7 +48,7 @@ struct FuncInfo { FuncInfo() = default; FuncInfo(Packet *p) { - int real_dims = p->dimensions / p->type.lanes; + int real_dims = p->dimensions / p->lanes; if (real_dims > 16) { fprintf(stderr, "Error: found trace packet with dimensionality > 16. Aborting.\n"); exit(1); @@ -58,16 +58,14 @@ struct FuncInfo { max_coords[i] = INT32_MIN; } dimensions = real_dims; - type = p->type; - type.lanes = 1; + type = p->type(); } void add_preprocess(Packet *p) { - int real_dims = p->dimensions / p->type.lanes; - int lanes = p->type.lanes; + int real_dims = p->dimensions / p->lanes; + int lanes = p->lanes; - halide_type_t scalar_type = p->type; - scalar_type.lanes = 1; + halide_type_t scalar_type = p->type(); if (scalar_type != type) { fprintf(stderr, "Error: packet type doesn't match previous packets of same Func. Aborting.\n"); @@ -101,8 +99,7 @@ struct FuncInfo { } void add(Packet *p) { - halide_type_t scalar_type = p->type; - scalar_type.lanes = 1; + halide_type_t scalar_type = p->type(); if (scalar_type == halide_type_of()) { add_typed(p); } else if (scalar_type == halide_type_of()) { @@ -134,7 +131,7 @@ struct FuncInfo { template void add_typed(Packet *p) { Buffer &buf = values.as(); - int lanes = p->type.lanes; + int lanes = p->lanes; if (!allocated()) { fprintf(stderr, "Packet storage not allocated. Aborting.\n"); diff --git a/util/HalideTraceUtils.h b/util/HalideTraceUtils.h index 20527c692f40..b06335d1350d 100644 --- a/util/HalideTraceUtils.h +++ b/util/HalideTraceUtils.h @@ -73,14 +73,14 @@ struct Packet : public halide_trace_packet_t { template T get_value_as(int idx) const { - const uint8_t *val = (const uint8_t *)(value()) + idx * type.bytes(); + const uint8_t *val = (const uint8_t *)(value()) + idx * type().bytes(); // 'val' may not be aligned: memcpy it to an aligned local // so that value_as<>() won't complain under sanitizers. halide_scalar_value_t aligned_value; // Only copy the number of bytes in the type: the stream isn't guaranteed // to be padded to sizeof(halide_scalar_value_t). - memcpy(&aligned_value, val, type.bits / 8); - return value_as(type, aligned_value); + memcpy(&aligned_value, val, type().bits / 8); + return value_as(type(), aligned_value); } // Grab a packet from stdin. Returns false when stdin closes. diff --git a/util/HalideTraceViz.cpp b/util/HalideTraceViz.cpp index e0f24687c174..19edebaa7d6b 100644 --- a/util/HalideTraceViz.cpp +++ b/util/HalideTraceViz.cpp @@ -107,28 +107,28 @@ struct fail { template T value_as(const halide_type_t &type, const halide_scalar_value_t &value) { - switch (type.element_of().as_u32()) { - case halide_type_t(halide_type_int, 8).as_u32(): + switch (type) { + case halide_type_t(halide_type_int, 8): return (T)value.u.i8; - case halide_type_t(halide_type_int, 16).as_u32(): + case halide_type_t(halide_type_int, 16): return (T)value.u.i16; - case halide_type_t(halide_type_int, 32).as_u32(): + case halide_type_t(halide_type_int, 32): return (T)value.u.i32; - case halide_type_t(halide_type_int, 64).as_u32(): + case halide_type_t(halide_type_int, 64): return (T)value.u.i64; - case halide_type_t(halide_type_uint, 1).as_u32(): + case halide_type_t(halide_type_uint, 1): return (T)value.u.b; - case halide_type_t(halide_type_uint, 8).as_u32(): + case halide_type_t(halide_type_uint, 8): return (T)value.u.u8; - case halide_type_t(halide_type_uint, 16).as_u32(): + case halide_type_t(halide_type_uint, 16): return (T)value.u.u16; - case halide_type_t(halide_type_uint, 32).as_u32(): + case halide_type_t(halide_type_uint, 32): return (T)value.u.u32; - case halide_type_t(halide_type_uint, 64).as_u32(): + case halide_type_t(halide_type_uint, 64): return (T)value.u.u64; - case halide_type_t(halide_type_float, 32).as_u32(): + case halide_type_t(halide_type_float, 32): return (T)value.u.f32; - case halide_type_t(halide_type_float, 64).as_u32(): + case halide_type_t(halide_type_float, 64): return (T)value.u.f64; default: fail() << "Can't convert packet with type: " << (int)type.code << "bits: " << type.bits; @@ -138,14 +138,15 @@ T value_as(const halide_type_t &type, const halide_scalar_value_t &value) { template T get_value_as(const halide_trace_packet_t &p, int idx) { - const uint8_t *val = (const uint8_t *)(p.value()) + idx * p.type.bytes(); + const halide_type_t type = p.type(); + const uint8_t *val = (const uint8_t *)(p.value()) + idx * type.bytes(); // 'val' may not be aligned: memcpy it to an aligned local // so that value_as<>() won't complain under sanitizers. halide_scalar_value_t aligned_value; // Only copy the number of bytes in the type: the stream isn't guaranteed // to be padded to sizeof(halide_scalar_value_t). - memcpy(&aligned_value, val, p.type.bits / 8); - return value_as(p.type, aligned_value); + memcpy(&aligned_value, val, type.bits / 8); + return value_as(type, aligned_value); } struct PacketAndPayload : public halide_trace_packet_t { @@ -214,19 +215,19 @@ struct FuncInfo { void observe_load(const halide_trace_packet_t &p) { observe_load_or_store(p); - loads += p.type.lanes; + loads += p.lanes; } void observe_store(const halide_trace_packet_t &p) { observe_load_or_store(p); - stores += p.type.lanes; + stores += p.lanes; } void observe_load_or_store(const halide_trace_packet_t &p) { const int *coords = p.coordinates(); - for (int i = 0; i < std::min(16, p.dimensions / p.type.lanes); i++) { - for (int lane = 0; lane < p.type.lanes; lane++) { - int coord = coords[i * p.type.lanes + lane]; + for (int i = 0; i < std::min(16, p.dimensions / p.lanes); i++) { + for (int lane = 0; lane < p.lanes; lane++) { + int coord = coords[i * p.lanes + lane]; if (loads + stores == 0 && lane == 0) { min_coord[i] = coord; max_coord[i] = coord + 1; @@ -237,7 +238,7 @@ struct FuncInfo { } } - for (int i = 0; i < p.type.lanes; i++) { + for (int i = 0; i < p.lanes; i++) { double value = get_value_as(p, i); if (stores + loads == 0) { min_value = value; @@ -1324,10 +1325,10 @@ int run(bool ignore_trace_tags, FlagProcessor flag_processor) { if (p.event == halide_trace_store) { // Stores take time proportional to the number of // items stored times the cost of the func. - halide_clock += fi.config.store_cost * p.type.lanes; + halide_clock += fi.config.store_cost * p.lanes; fi.stats.observe_store(p); } else { - halide_clock += fi.config.load_cost * p.type.lanes; + halide_clock += fi.config.load_cost * p.lanes; fi.stats.observe_load(p); } @@ -1336,15 +1337,15 @@ int run(bool ignore_trace_tags, FlagProcessor flag_processor) { // fi.config.strides are provided by the --stride flag, so it can contain anything; i // if you don't specify them at all, they default to {{1,0},{0,1} (aka size=2). // So if we have excess strides, just ignore them. - const int dims = std::min(p.dimensions / p.type.lanes, (int)fi.config.strides.size()); + const int dims = std::min(p.dimensions / p.lanes, (int)fi.config.strides.size()); const int *coords = p.coordinates(); - for (int lane = 0; lane < p.type.lanes; lane++) { + for (int lane = 0; lane < p.lanes; lane++) { // Compute the screen-space x, y coord to draw this. int x = fi.config.pos.x; int y = fi.config.pos.y; const float z = fi.config.zoom; for (int d = 0; d < dims; d++) { - const int coord = d * p.type.lanes + lane; + const int coord = d * p.lanes + lane; assert(coord < p.dimensions); const int a = coords[coord]; const auto &stride = fi.config.strides[d]; @@ -1381,7 +1382,7 @@ int run(bool ignore_trace_tags, FlagProcessor flag_processor) { image_color = (int_value * 0x00010101) | 0xff000000; } else { // Color - uint32_t channel = coords[fi.config.color_dim * p.type.lanes + lane]; + uint32_t channel = coords[fi.config.color_dim * p.lanes + lane]; uint32_t mask = ~(255 << (channel * 8)); image_color &= mask; image_color |= int_value << (channel * 8); From 1c50698f9d8f58f7cae9884466d62f37ebc30c86 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Thu, 16 Jul 2026 17:35:49 -0400 Subject: [PATCH 4/4] Disable generator_jit_{stubtest,configure,example} on WASM --- test/generator/CMakeLists.txt | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/generator/CMakeLists.txt b/test/generator/CMakeLists.txt index 92dd9bc52ac3..efc157f408f0 100644 --- a/test/generator/CMakeLists.txt +++ b/test/generator/CMakeLists.txt @@ -786,9 +786,11 @@ _add_halide_aot_tests(string_param) add_halide_generator(stubtest.generator SOURCES stubtest_generator.cpp) add_halide_stub(stubtest.stub FROM stubtest.generator GENERATOR stubtest) -add_executable(generator_jit_stubtest stubtest_jittest.cpp stubtest_generator.cpp) -target_link_libraries(generator_jit_stubtest PRIVATE Halide::Test stubtest.stub) -add_halide_test(generator_jit_stubtest GROUPS generator) +if (NOT ${_USING_WASM}) + add_executable(generator_jit_stubtest stubtest_jittest.cpp stubtest_generator.cpp) + target_link_libraries(generator_jit_stubtest PRIVATE Halide::Test stubtest.stub) + add_halide_test(generator_jit_stubtest GROUPS generator) +endif () _add_halide_libraries( stubtest @@ -836,15 +838,19 @@ _add_halide_aot_tests( # configure_jittest.cpp # (configure.stub was already created FROM configure.generator, above, for stubuser) -add_executable(generator_jit_configure configure_jittest.cpp configure_generator.cpp) -target_link_libraries(generator_jit_configure PRIVATE Halide::Test configure.stub) -add_halide_test(generator_jit_configure GROUPS generator) +if (NOT ${_USING_WASM}) + add_executable(generator_jit_configure configure_jittest.cpp configure_generator.cpp) + target_link_libraries(generator_jit_configure PRIVATE Halide::Test configure.stub) + add_halide_test(generator_jit_configure GROUPS generator) +endif () # example_jittest.cpp -add_halide_stub(example.stub FROM example.generator GENERATOR example) -add_executable(generator_jit_example example_jittest.cpp example_generator.cpp) -target_link_libraries(generator_jit_example PRIVATE Halide::Test example.stub) -add_halide_test(generator_jit_example GROUPS generator) +if (NOT ${_USING_WASM}) + add_halide_stub(example.stub FROM example.generator GENERATOR example) + add_executable(generator_jit_example example_jittest.cpp example_generator.cpp) + target_link_libraries(generator_jit_example PRIVATE Halide::Test example.stub) + add_halide_test(generator_jit_example GROUPS generator) +endif () if (NOT ${_USING_WASM}) # rungen_test.cpp